Merge pull request #78 from bnema/perf/transport-observability

perf(obs): add transport runtime measurement harness

commit: f106d97fc96812b53243cdc4fc249d9e6bcdae85
author: bnema · 3mo ago · 2026-07-13 14:24 UTC
committer: GitHub · 3mo ago · 2026-07-13 14:24 UTC
merge of: 187a82b + 6b1947d · diff against the first parent
signature: GPG UnverifiedB5690EEEBB952194 The key is unknown

46 files · +9917 -201

A cmd/vev-perf-harness/evidence.go +324 -0

@@ -0,0 +1,324 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"os"
	9	+	"sort"
	10	+	"strings"
	11	+	)
	12	+
	13	+	func measuredEventSamples(marks []harnessMark, interval measuredInterval) ([]eventSample, error) {
	14	+	if interval.End < interval.Start {
	15	+	return nil, errors.New("negative measured interval")
	16	+	}
	17	+	starts := map[uint64]int64{}
	18	+	events := make([]eventSample, 0, len(marks)/2)
	19	+	for _, m := range marks {
	20	+	if !m.Valid {
	21	+	return nil, errors.New("invalid harness boundary")
	22	+	}
	23	+	switch m.Kind {
	24	+	case "input_injected":
	25	+	if _, exists := starts[m.Sequence]; exists {
	26	+	return nil, errors.New("duplicate input boundary")
	27	+	}
	28	+	starts[m.Sequence] = m.Tick
	29	+	case "terminal_flushed":
	30	+	start, ok := starts[m.Sequence]
	31	+	if !ok {
	32	+	return nil, errors.New("terminal flush without input pair")
	33	+	}
	34	+	if m.Tick < start {
	35	+	return nil, errors.New("negative harness duration")
	36	+	}
	37	+	delete(starts, m.Sequence)
	38	+	if start >= interval.Start && m.Tick < interval.End {
	39	+	events = append(events, eventSample{Sequence: m.Sequence, Injected: start, Flushed: m.Tick, Latency: m.Tick - start})
	40	+	}
	41	+	default:
	42	+	return nil, errors.New("unknown harness mark")
	43	+	}
	44	+	}
	45	+	if len(starts) != 0 {
	46	+	return nil, errors.New("missing terminal flush pair")
	47	+	}
	48	+	sort.Slice(events, func(i, j int) bool {
	49	+	if events[i].Injected == events[j].Injected {
	50	+	return events[i].Sequence < events[j].Sequence
	51	+	}
	52	+	return events[i].Injected < events[j].Injected
	53	+	})
	54	+	return events, nil
	55	+	}
	56	+
	57	+	func requireMinimumEventSamples(events []eventSample) error {
	58	+	if len(events) < minimumInIntervalEventSamples {
	59	+	return fmt.Errorf("insufficient in-interval event samples: got %d, need at least %d", len(events), minimumInIntervalEventSamples)
	60	+	}
	61	+	return nil
	62	+	}
	63	+
	64	+	func eventLatencies(events []eventSample) []int64 {
	65	+	out := make([]int64, len(events))
	66	+	for i, event := range events {
	67	+	out[i] = event.Latency
	68	+	}
	69	+	return out
	70	+	}
	71	+
	72	+	func eventCadenceSamples(events []eventSample) ([]int64, int64) {
	73	+	if len(events) < 2 {
	74	+	return nil, 0
	75	+	}
	76	+	gaps := make([]int64, 0, len(events)-1)
	77	+	var maxGap int64
	78	+	for i := 1; i < len(events); i++ {
	79	+	gap := events[i].Injected - events[i-1].Injected
	80	+	gaps = append(gaps, gap)
	81	+	if gap > maxGap {
	82	+	maxGap = gap
	83	+	}
	84	+	}
	85	+	return gaps, maxGap
	86	+	}
	87	+
	88	+	func eventCadence(events []eventSample) (distribution, int64) {
	89	+	gaps, maxGap := eventCadenceSamples(events)
	90	+	if len(gaps) == 0 {
	91	+	return distribution{}, maxGap
	92	+	}
	93	+	return percentiles(gaps), maxGap
	94	+	}
	95	+
	96	+	func pairedSamples(marks []harnessMark) ([]int64, error) {
	97	+	starts := map[uint64]int64{}
	98	+	var out []int64
	99	+	for _, m := range marks {
	100	+	if !m.Valid {
	101	+	return nil, errors.New("invalid harness boundary")
	102	+	}
	103	+	switch m.Kind {
	104	+	case "input_injected":
	105	+	if _, ok := starts[m.Sequence]; ok {
	106	+	return nil, errors.New("duplicate input boundary")
	107	+	}
	108	+	starts[m.Sequence] = m.Tick
	109	+	case "terminal_flushed":
	110	+	start, ok := starts[m.Sequence]
	111	+	if !ok {
	112	+	return nil, errors.New("terminal flush without input pair")
	113	+	}
	114	+	if m.Tick < start {
	115	+	return nil, errors.New("negative harness duration")
	116	+	}
	117	+	out = append(out, m.Tick-start)
	118	+	delete(starts, m.Sequence)
	119	+	default:
	120	+	return nil, errors.New("unknown harness mark")
	121	+	}
	122	+	}
	123	+	if len(starts) != 0 {
	124	+	return nil, errors.New("missing terminal flush pair")
	125	+	}
	126	+	if len(out) == 0 {
	127	+	return nil, errors.New("insufficient harness samples")
	128	+	}
	129	+	return out, nil
	130	+	}
	131	+
	132	+	// traceRecord is intentionally local to the command. It mirrors JSONL fields
	133	+	// needed for correlation but never exposes a clock to any production component.
	134	+	type traceRecord struct {
	135	+	ProcessID string `json:"process_id"`
	136	+	Component string `json:"component"`
	137	+	Scenario string `json:"scenario"`
	138	+	Run uint64 `json:"run"`
	139	+	Sequence uint64 `json:"sequence"`
	140	+	RequestID uint64 `json:"request_id"`
	141	+	Epoch uint64 `json:"epoch"`
	142	+	Kind string `json:"kind"`
	143	+	Tick int64 `json:"tick"`
	144	+	Valid bool `json:"valid"`
	145	+	}
	146	+
	147	+	type spanPair struct{ start, end, name string }
	148	+
	149	+	// mergeProcessTraces permits only records from one manifest process in a span.
	150	+	// In particular, the correlation key includes process_id before ticks are read.
	151	+	func mergeProcessTraces(mappings []processMapping) ([]span, error) {
	152	+	known := map[string]processMapping{}
	153	+	for _, m := range mappings {
	154	+	if m.ProcessID == "" \|\| m.ClockDomain == "" \|\| m.TracePath == "" \|\| known[m.ProcessID].ProcessID != "" {
	155	+	return nil, errors.New("duplicate or incomplete manifest process mapping")
	156	+	}
	157	+	known[m.ProcessID] = m
	158	+	}
	159	+	out := []span{}
	160	+	for _, m := range mappings {
	161	+	f, err := os.Open(m.TracePath)
	162	+	if err != nil {
	163	+	return nil, fmt.Errorf("open manifest trace for %s role: %w", m.Role, err)
	164	+	}
	165	+	var records []traceRecord
	166	+	scan := bufio.NewScanner(f)
	167	+	for scan.Scan() {
	168	+	var r traceRecord
	169	+	if err := json.Unmarshal(scan.Bytes(), &r); err != nil {
	170	+	return nil, closeFile(f, err)
	171	+	}
	172	+	if r.ProcessID != m.ProcessID \|\| r.Scenario != m.Scenario \|\| r.Run != uint64(m.Run) {
	173	+	return nil, closeFile(f, errors.New("trace record identity does not match manifest"))
	174	+	}
	175	+	if r.Scenario == "" \|\| r.Run == 0 \|\| r.Component == "" \|\| r.Sequence == 0 \|\| r.RequestID == 0 \|\| r.Epoch == 0 {
	176	+	return nil, closeFile(f, errors.New("trace record has invalid correlation fields"))
	177	+	}
	178	+	if _, ok := known[r.ProcessID]; !ok {
	179	+	return nil, closeFile(f, errors.New("unknown trace process_id"))
	180	+	}
	181	+	records = append(records, r)
	182	+	}
	183	+	if err := scan.Err(); err != nil {
	184	+	return nil, closeFile(f, err)
	185	+	}
	186	+	if err := f.Close(); err != nil {
	187	+	return nil, err
	188	+	}
	189	+	// A process shares one observer across concurrent components. Correlation
	190	+	// IDs are allocated before a goroutine reaches its first mark, so a later
	191	+	// ID can be serialized before an earlier one. Their first appearance is
	192	+	// consequently not an ordering contract. Only marks in the same component
	193	+	// and exact correlation domain may pair, and each pair's ticks must retain
	194	+	// its own start-before-end ordering.
	195	+	starts := map[string]traceRecord{}
	196	+	for _, r := range records {
	197	+	for _, pair := range spanPairs {
	198	+	key := fmt.Sprintf("%s/%s/%s/%d/%d/%d/%d", r.ProcessID, r.Component, m.Scenario, m.Run, r.Sequence, r.RequestID, r.Epoch)
	199	+	if r.Kind == pair.start {
	200	+	k := pair.name + "/" + key
	201	+	if _, exists := starts[k]; exists {
	202	+	return nil, errors.New("duplicate process-local span start")
	203	+	}
	204	+	starts[k] = r
	205	+	}
	206	+	if r.Kind == pair.end {
	207	+	k := pair.name + "/" + key
	208	+	start, ok := starts[k]
	209	+	if !ok {
	210	+	return nil, errors.New("span end without same-process start")
	211	+	}
	212	+	if r.Tick < start.Tick {
	213	+	return nil, errors.New("negative same-process span")
	214	+	}
	215	+	// Validity affects only measurement eligibility, never structural
	216	+	// pairing: every start/end must still match in its exact process,
	217	+	// component, and correlation domain. A failed start, failed end, or
	218	+	// both is a paired diagnostic fact but contributes no latency sample.
	219	+	if start.Valid && r.Valid {
	220	+	out = append(out, span{Component: r.Component, Name: pair.name, Samples: []int64{r.Tick - start.Tick}})
	221	+	}
	222	+	delete(starts, k)
	223	+	}
	224	+	}
	225	+	}
	226	+	if len(starts) != 0 {
	227	+	keys := make([]string, 0, len(starts))
	228	+	for key := range starts {
	229	+	keys = append(keys, key)
	230	+	}
	231	+	sort.Strings(keys)
	232	+	return nil, fmt.Errorf("missing process-local span pair for %s: %s", m.ProcessID, strings.Join(keys, ", "))
	233	+	}
	234	+	}
	235	+	return out, nil
	236	+	}
	237	+
	238	+	func summarizeSpans(all []span) []spanSummary {
	239	+	grouped := map[string]*span{}
	240	+	for _, s := range all {
	241	+	k := s.Component + "\x00" + s.Name
	242	+	if grouped[k] == nil {
	243	+	grouped[k] = &span{Component: s.Component, Name: s.Name}
	244	+	}
	245	+	grouped[k].Samples = append(grouped[k].Samples, s.Samples...)
	246	+	}
	247	+	keys := make([]string, 0, len(grouped))
	248	+	for k := range grouped {
	249	+	keys = append(keys, k)
	250	+	}
	251	+	sort.Strings(keys)
	252	+	out := make([]spanSummary, 0, len(keys))
	253	+	for _, k := range keys {
	254	+	s := grouped[k]
	255	+	if len(s.Samples) < minimumRepetitions {
	256	+	out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: distribution{Count: len(s.Samples)}})
	257	+	continue
	258	+	}
	259	+	out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: percentiles(s.Samples)})
	260	+	}
	261	+	return out
	262	+	}
	263	+
	264	+	func percentiles(samples []int64) distribution {
	265	+	v := append([]int64(nil), samples...)
	266	+	sort.Slice(v, func(i, j int) bool { return v[i] < v[j] })
	267	+	at := func(p int) int64 { return v[(len(v)-1)*p/100] }
	268	+	return distribution{len(v), at(50), at(95), at(99), v[len(v)-1]}
	269	+	}
	270	+
	271	+	func closeFile(f *os.File, err error) error {
	272	+	return errors.Join(err, f.Close())
	273	+	}
	274	+
	275	+	func writeJSON(path string, v any) error {
	276	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
	277	+	if err != nil {
	278	+	return err
	279	+	}
	280	+	if err := json.NewEncoder(f).Encode(v); err != nil {
	281	+	return closeFile(f, err)
	282	+	}
	283	+	return f.Close()
	284	+	}
	285	+
	286	+	func mustJSON(v any) string {
	287	+	b, err := json.Marshal(v)
	288	+	if err != nil {
	289	+	panic(fmt.Sprintf("marshal JSON: %v", err))
	290	+	}
	291	+	return string(b)
	292	+	}
	293	+
	294	+	func safeName(s string) string {
	295	+	return strings.Map(func(r rune) rune {
	296	+	if r >= 'a' && r <= 'z' \|\| r >= 'A' && r <= 'Z' \|\| r >= '0' && r <= '9' \|\| r == '-' \|\| r == '_' {
	297	+	return r
	298	+	}
	299	+	return '-'
	300	+	}, s)
	301	+	}
	302	+
	303	+	func readJSONL(path string) ([]map[string]json.RawMessage, error) {
	304	+	f, err := os.Open(path)
	305	+	if err != nil {
	306	+	return nil, err
	307	+	}
	308	+	var r []map[string]json.RawMessage
	309	+	scan := bufio.NewScanner(f)
	310	+	for scan.Scan() {
	311	+	var v map[string]json.RawMessage
	312	+	if err := json.Unmarshal(scan.Bytes(), &v); err != nil {
	313	+	return nil, closeFile(f, err)
	314	+	}
	315	+	r = append(r, v)
	316	+	}
	317	+	if err := scan.Err(); err != nil {
	318	+	return nil, closeFile(f, err)
	319	+	}
	320	+	if err := f.Close(); err != nil {
	321	+	return nil, err
	322	+	}
	323	+	return r, nil
	324	+	}

A cmd/vev-perf-harness/evidence_test.go +387 -0

@@ -0,0 +1,387 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"os"
	8	+	"path/filepath"
	9	+	"strings"
	10	+	"testing"
	11	+	"time"
	12	+	)
	13	+
	14	+	func TestCLITransportSeamOwnsExclusivePeerTraceAndCleanup(t *testing.T) {
	15	+	fixtures := []struct {
	16	+	name string
	17	+	tr transport
	18	+	rtt time.Duration
	19	+	loss int
	20	+	}{
	21	+	{"baseline", transport{ID: "udp_baseline", Kind: "udp"}, 0, 0},
	22	+	{"25ms", transport{ID: "udp_25ms", Kind: "udp", RTTMS: 25}, 25 * time.Millisecond, 0},
	23	+	{"100ms", transport{ID: "udp_100ms", Kind: "udp", RTTMS: 100}, 100 * time.Millisecond, 0},
	24	+	{"loss0", transport{ID: "udp_loss_0pct", Kind: "udp"}, 0, 0},
	25	+	{"loss1", transport{ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1}, 0, 1},
	26	+	}
	27	+	for _, tc := range fixtures {
	28	+	t.Run(tc.name, func(t *testing.T) {
	29	+	dir := t.TempDir()
	30	+	if err := os.Chmod(dir, 0o700); err != nil {
	31	+	t.Fatal(err)
	32	+	}
	33	+	var configs []udpNetemConfig
	34	+	netem := &fakeUDPNetem{port: 45678}
	35	+	l := &cliLauncher{bin: "/bin/true", netemFactory: func(c udpNetemConfig) (udpNetem, error) {
	36	+	configs = append(configs, c)
	37	+	return netem, nil
	38	+	}}
	39	+	m := processMapping{ProcessID: "udp-peer", TracePath: filepath.Join(dir, "udp.jsonl"), Role: "udp_peer"}
	40	+	if err := os.WriteFile(m.TracePath, nil, 0o600); err != nil {
	41	+	t.Fatal(err)
	42	+	}
	43	+	p, err := l.preparePeer(m, roleCommand{Args: []string{"_udp-proxy", "work"}, Transport: tc.tr})
	44	+	if err != nil {
	45	+	t.Fatal(err)
	46	+	}
	47	+	if len(configs) != 1 \|\| configs[0].RTT != tc.rtt \|\| configs[0].LossPercent != tc.loss \|\| configs[0].TargetPath != filepath.Join(dir, "udp-peer.target") {
	48	+	t.Fatalf("fixture did not reach netem seam: %+v", configs)
	49	+	}
	50	+	shim, err := os.ReadFile(filepath.Join(dir, "ssh"))
	51	+	if err != nil {
	52	+	t.Fatal(err)
	53	+	}
	54	+	text := string(shim)
	55	+	for _, want := range []string{"_udp-proxy", m.TracePath, m.ProcessID, "udp-peer.target", "VEV-UDP %s %s\\n' 45678", "udp-peer.pid"} {
	56	+	if !strings.Contains(text, want) {
	57	+	t.Errorf("seam does not retain %q:\n%s", want, text)
	58	+	}
	59	+	}
	60	+	if strings.Contains(text, "VEV_PERF_UDP_") {
	61	+	t.Fatalf("ignored vev UDP environment was used instead of netem: %s", text)
	62	+	}
	63	+	// A nonexistent pid is already-cleaned-up; Close must still close the
	64	+	// harness-owned emulator.
	65	+	if err := p.Close(); err != nil {
	66	+	t.Fatal(err)
	67	+	}
	68	+	if !netem.closed {
	69	+	t.Fatal("harness netem was not cleaned up")
	70	+	}
	71	+	})
	72	+	}
	73	+	}
	74	+
	75	+	func TestHarnessRejectsInvalidBoundaryPairs(t *testing.T) {
	76	+	for _, marks := range [][]harnessMark{
	77	+	{{Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
	78	+	{{Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}, {Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
	79	+	{{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}},
	80	+	{{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}},
	81	+	} {
	82	+	if _, err := pairedSamples(marks); err == nil {
	83	+	t.Fatalf("invalid marks accepted: %+v", marks)
	84	+	}
	85	+	}
	86	+	got, err := pairedSamples([]harnessMark{{Sequence: 9, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 9, Kind: "terminal_flushed", Tick: 5, Valid: true}})
	87	+	if err != nil \|\| len(got) != 1 \|\| got[0] != 4 {
	88	+	t.Fatalf("got %v %v", got, err)
	89	+	}
	90	+	}
	91	+
	92	+	func TestHarnessCollectsOnlyCompleteMeasuredIntervalEvents(t *testing.T) {
	93	+	interval := measuredInterval{Start: 100, End: 200}
	94	+	marks := []harnessMark{
	95	+	{Sequence: 1, Kind: "input_injected", Tick: 10, Valid: true},
	96	+	{Sequence: 1, Kind: "terminal_flushed", Tick: 20, Valid: true}, // warmup
	97	+	{Sequence: 2, Kind: "input_injected", Tick: 100, Valid: true},
	98	+	{Sequence: 2, Kind: "terminal_flushed", Tick: 120, Valid: true},
	99	+	{Sequence: 3, Kind: "input_injected", Tick: 150, Valid: true},
	100	+	{Sequence: 3, Kind: "terminal_flushed", Tick: 201, Valid: true}, // exits interval
	101	+	{Sequence: 4, Kind: "input_injected", Tick: 99, Valid: true},
	102	+	{Sequence: 4, Kind: "terminal_flushed", Tick: 110, Valid: true}, // enters interval
	103	+	{Sequence: 5, Kind: "input_injected", Tick: 180, Valid: true},
	104	+	{Sequence: 5, Kind: "terminal_flushed", Tick: 200, Valid: true}, // end is excluded
	105	+	}
	106	+	events, err := measuredEventSamples(marks, interval)
	107	+	if err != nil {
	108	+	t.Fatal(err)
	109	+	}
	110	+	if len(events) != 1 \|\| events[0].Sequence != 2 \|\| events[0].Latency != 20 {
	111	+	t.Fatalf("measured events=%+v, want only complete sequence 2", events)
	112	+	}
	113	+	}
	114	+
	115	+	func TestHarnessRejectsInsufficientMeasuredEventSamples(t *testing.T) {
	116	+	marks := make([]harnessMark, 0, 2*(minimumInIntervalEventSamples-1))
	117	+	for i := 0; i < minimumInIntervalEventSamples-1; i++ {
	118	+	marks = append(marks,
	119	+	harnessMark{Sequence: uint64(i + 1), Kind: "input_injected", Tick: int64(i * 100), Valid: true},
	120	+	harnessMark{Sequence: uint64(i + 1), Kind: "terminal_flushed", Tick: int64(i*100 + 10), Valid: true},
	121	+	)
	122	+	}
	123	+	events, err := measuredEventSamples(marks, measuredInterval{Start: 0, End: int64(minimumInIntervalEventSamples * 100)})
	124	+	if err != nil {
	125	+	t.Fatal(err)
	126	+	}
	127	+	if err := requireMinimumEventSamples(events); err == nil {
	128	+	t.Fatal("insufficient measured event samples accepted")
	129	+	}
	130	+	}
	131	+
	132	+	func TestHarnessRunRejectsStraddlingEventsAsInsufficient(t *testing.T) {
	133	+	dir := t.TempDir()
	134	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	135	+	if err != nil {
	136	+	t.Fatal(err)
	137	+	}
	138	+	t.Cleanup(func() {
	139	+	if err := raw.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
	140	+	t.Error(err)
	141	+	}
	142	+	})
	143	+	clock := &fakeClock{}
	144	+	h := defaultHarness()
	145	+	h.clock, h.launcher = clock, insufficientEventLauncher{clock: clock}
	146	+	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "late", Roles: []string{"daemon", "client"}}, 1, raw)
	147	+	if err == nil \|\| !strings.Contains(err.Error(), "insufficient in-interval event samples") {
	148	+	t.Fatalf("straddling run error=%v, want insufficient in-interval sample rejection", err)
	149	+	}
	150	+	if err := raw.Close(); err != nil {
	151	+	t.Fatal(err)
	152	+	}
	153	+	marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
	154	+	if got, want := len(marks), 4; got != want {
	155	+	t.Fatalf("raw marks=%d, want warmup and straddling pair=%d: %+v", got, want, marks)
	156	+	}
	157	+	if straddling := marks[len(marks)-1]; straddling.Sequence != 2 \|\| straddling.Kind != "terminal_flushed" \|\| !straddling.Valid {
	158	+	t.Fatalf("straddling boundary=%+v", straddling)
	159	+	}
	160	+	}
	161	+
	162	+	func TestHarnessCalculatesEventCadenceAndMaxGapFromRawPairs(t *testing.T) {
	163	+	events := []eventSample{
	164	+	{Sequence: 1, Injected: 100, Flushed: 110, Latency: 10},
	165	+	{Sequence: 2, Injected: 130, Flushed: 140, Latency: 10},
	166	+	{Sequence: 3, Injected: 180, Flushed: 190, Latency: 10},
	167	+	}
	168	+	cadence, maxGap := eventCadence(events)
	169	+	if cadence.Count != 2 \|\| cadence.P50 != 30 \|\| cadence.P95 != 30 \|\| cadence.P99 != 30 \|\| maxGap != 50 {
	170	+	t.Fatalf("cadence=%+v maxGap=%d", cadence, maxGap)
	171	+	}
	172	+	}
	173	+
	174	+	func TestHarnessIncludesAllRequiredProcessLocalSpans(t *testing.T) {
	175	+	got := make([]string, 0, len(spanPairs))
	176	+	for _, pair := range spanPairs {
	177	+	got = append(got, pair.name)
	178	+	}
	179	+	want := []string{"capture_duration", "compose_duration", "diff_duration", "queue_wait", "ack_blocked_interval", "emit_duration", "adapter_send_duration", "adapter_receive_duration"}
	180	+	if !equalStrings(got, want) {
	181	+	t.Fatalf("span pairs = %v, want %v", got, want)
	182	+	}
	183	+	}
	184	+
	185	+	func TestHarnessRawRecordsAreDeterministicJSONL(t *testing.T) {
	186	+	var b bytes.Buffer
	187	+	for _, m := range []harnessMark{{Scenario: "s", Run: 1, Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Scenario: "s", Run: 1, Sequence: 1, Kind: "terminal_flushed", Tick: 2, Valid: true}} {
	188	+	b.WriteString(mustJSON(m))
	189	+	b.WriteByte('\n')
	190	+	}
	191	+	path := filepath.Join(t.TempDir(), "raw.jsonl")
	192	+	if err := os.WriteFile(path, b.Bytes(), 0o600); err != nil {
	193	+	t.Fatal(err)
	194	+	}
	195	+	if got, err := readJSONL(path); err != nil \|\| len(got) != 2 {
	196	+	t.Fatalf("records=%d err=%v", len(got), err)
	197	+	}
	198	+	}
	199	+
	200	+	func TestHarnessClosesRolesBeforeMergingReceiveSpans(t *testing.T) {
	201	+	dir := t.TempDir()
	202	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	203	+	if err != nil {
	204	+	t.Fatal(err)
	205	+	}
	206	+	t.Cleanup(func() {
	207	+	if err := raw.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
	208	+	t.Error(err)
	209	+	}
	210	+	})
	211	+	launcher := &closeTraceLauncher{}
	212	+	h := defaultHarness()
	213	+	h.clock, h.launcher = &fakeClock{}, launcher
	214	+	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "receive-cleanup", Roles: []string{"daemon", "client"}}, 1, raw)
	215	+	if err != nil {
	216	+	t.Fatal(err)
	217	+	}
	218	+	if len(result.Spans) != 2 {
	219	+	t.Fatalf("post-cleanup receive spans=%+v", result.Spans)
	220	+	}
	221	+	for _, s := range result.Spans {
	222	+	if s.Name != "adapter_receive_duration" \|\| len(s.Samples) != 1 \|\| s.Samples[0] != 10 {
	223	+	t.Fatalf("unmatched or wrong receive span: %+v", s)
	224	+	}
	225	+	}
	226	+	for i, p := range launcher.process {
	227	+	if !p.closed {
	228	+	t.Fatalf("role %s was not closed before trace merge", launcher.mappings[i].Role)
	229	+	}
	230	+	}
	231	+	}
	232	+
	233	+	func TestHarnessWritesRequiredEvidenceWithSufficientSpans(t *testing.T) {
	234	+	base, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
	235	+	if err != nil {
	236	+	t.Fatal(err)
	237	+	}
	238	+	for i := range base.Scenarios {
	239	+	if i > 0 {
	240	+	base.Scenarios[i].InapplicableReason = "fixture limits this test to one public topology"
	241	+	base.Scenarios[i].Roles = nil
	242	+	}
	243	+	}
	244	+	dir := t.TempDir()
	245	+	manifestPath := filepath.Join(dir, "manifest.json")
	246	+	if err := os.WriteFile(manifestPath, []byte(mustJSON(base)), 0o600); err != nil {
	247	+	t.Fatal(err)
	248	+	}
	249	+	l := &traceLauncher{}
	250	+	h := defaultHarness()
	251	+	h.clock = &fakeClock{}
	252	+	h.launcher = l
	253	+	h.gitSHA = func() string { return "test-sha" }
	254	+	if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", filepath.Join(dir, "out"), "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
	255	+	t.Fatal(err)
	256	+	}
	257	+	for _, name := range []string{"raw-harness.jsonl", "runs.json", "summary.json"} {
	258	+	if _, err := os.Stat(filepath.Join(dir, "out", name)); err != nil {
	259	+	t.Errorf("%s: %v", name, err)
	260	+	}
	261	+	}
	262	+	var got summary
	263	+	b, err := os.ReadFile(filepath.Join(dir, "out", "summary.json"))
	264	+	if err != nil {
	265	+	t.Fatal(err)
	266	+	}
	267	+	if err := json.Unmarshal(b, &got); err != nil {
	268	+	t.Fatal(err)
	269	+	}
	270	+	if got.EndToEnd.Count < 10*minimumInIntervalEventSamples \|\| got.Repetitions != 10 \|\| got.RunP50Dispersion.Count != 10 \|\| got.Cadence.Count == 0 \|\| got.MaxGap == 0 \|\| len(got.Spans) == 0 {
	271	+	t.Fatalf("summary=%+v", got)
	272	+	}
	273	+	for _, s := range got.Spans {
	274	+	if s.Distribution.Count < 10 {
	275	+	t.Fatalf("insufficient span summary: %+v", s)
	276	+	}
	277	+	}
	278	+	}
	279	+
	280	+	func TestHarnessAcceptsConcurrentProductionTraceInterleaving(t *testing.T) {
	281	+	path := filepath.Join(t.TempDir(), "local-client.jsonl")
	282	+	if err := os.WriteFile(path, []byte(boundedLocalConcurrentTrace), 0o600); err != nil {
	283	+	t.Fatal(err)
	284	+	}
	285	+	spans, err := mergeProcessTraces([]processMapping{{
	286	+	ProcessID: "1x4-idle-local-r002-client-02", ClockDomain: "1x4-idle-local-r002-client-02", TracePath: path,
	287	+	Scenario: "1x4-idle-local", Run: 2,
	288	+	}})
	289	+	if err != nil {
	290	+	t.Fatalf("valid concurrent production trace rejected: %v", err)
	291	+	}
	292	+	if len(spans) != 2 \|\| spans[0].Component != "ipc" \|\| spans[0].Name != "adapter_send_duration" \|\| spans[0].Samples[0] != 21900 \|\| spans[1].Component != "ipc" \|\| spans[1].Name != "adapter_receive_duration" \|\| spans[1].Samples[0] != 178550 {
	293	+	t.Fatalf("spans=%+v", spans)
	294	+	}
	295	+	}
	296	+
	297	+	func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {
	298	+	write := func(t *testing.T, records ...traceRecord) processMapping {
	299	+	t.Helper()
	300	+	path := filepath.Join(t.TempDir(), "trace.jsonl")
	301	+	f, err := os.Create(path)
	302	+	if err != nil {
	303	+	t.Fatal(err)
	304	+	}
	305	+	for _, r := range records {
	306	+	b, err := json.Marshal(r)
	307	+	if err != nil {
	308	+	t.Fatal(err)
	309	+	}
	310	+	if _, err := f.Write(append(b, '\n')); err != nil {
	311	+	t.Fatal(err)
	312	+	}
	313	+	}
	314	+	if err := f.Close(); err != nil {
	315	+	t.Fatal(err)
	316	+	}
	317	+	return processMapping{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}
	318	+	}
	319	+	base := func(kind string, tick int64) traceRecord {
	320	+	return traceRecord{ProcessID: "one", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: true}
	321	+	}
	322	+	for _, records := range [][]traceRecord{
	323	+	{base("diff_end", 1)},
	324	+	{base("diff_start", 2), base("diff_end", 1)},
	325	+	{base("diff_start", 1)},
	326	+	{base("diff_start", 1), base("diff_start", 2)},
	327	+	{base("diff_start", 1), func() traceRecord { r := base("diff_end", 2); r.Component = "ipc"; return r }()},
	328	+	{func() traceRecord { r := base("diff_start", 1); r.ProcessID = "other"; return r }()},
	329	+	{func() traceRecord { r := base("diff_start", 1); r.Scenario = "other-scenario"; return r }()},
	330	+	{func() traceRecord { r := base("diff_start", 1); r.Run = 2; return r }()},
	331	+	} {
	332	+	if _, err := mergeProcessTraces([]processMapping{write(t, records...)}); err == nil {
	333	+	t.Fatalf("invalid trace accepted: %+v", records)
	334	+	}
	335	+	}
	336	+	m := write(t, base("diff_start", 1), base("diff_end", 3))
	337	+	spans, err := mergeProcessTraces([]processMapping{m})
	338	+	if err != nil \|\| len(spans) != 1 \|\| spans[0].Samples[0] != 2 {
	339	+	t.Fatalf("spans=%+v err=%v", spans, err)
	340	+	}
	341	+	}
	342	+
	343	+	func TestHarnessExcludesFailedSpanDurationsWhileValidatingPairing(t *testing.T) {
	344	+	base := func(kind string, tick int64, valid bool) traceRecord {
	345	+	return traceRecord{ProcessID: "one", Component: "adapter", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: valid}
	346	+	}
	347	+	for _, tc := range []struct {
	348	+	name string
	349	+	records []traceRecord
	350	+	wantSpans int
	351	+	wantErr bool
	352	+	}{
	353	+	{"successful adapter send is sampled", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, true)}, 1, false},
	354	+	{"failed adapter send end is excluded", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, false)}, 0, false},
	355	+	{"failed adapter receive end is excluded", []traceRecord{base("adapter_receive_start", 10, true), base("adapter_receive_end", 20, false)}, 0, false},
	356	+	{"failed adapter send start is excluded", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_end", 20, true)}, 0, false},
	357	+	{"failed adapter receive boundaries are excluded", []traceRecord{base("adapter_receive_start", 10, false), base("adapter_receive_end", 20, false)}, 0, false},
	358	+	{"failed end without start is rejected", []traceRecord{base("adapter_send_end", 20, false)}, 0, true},
	359	+	{"duplicate failed start is rejected", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_start", 15, false)}, 0, true},
	360	+	{"negative failed span is rejected", []traceRecord{base("adapter_receive_start", 20, false), base("adapter_receive_end", 10, false)}, 0, true},
	361	+	{"cross-component failed end is rejected", []traceRecord{base("adapter_send_start", 10, true), func() traceRecord { r := base("adapter_send_end", 20, false); r.Component = "other"; return r }()}, 0, true},
	362	+	} {
	363	+	t.Run(tc.name, func(t *testing.T) {
	364	+	path := filepath.Join(t.TempDir(), "trace.jsonl")
	365	+	var lines []string
	366	+	for _, record := range tc.records {
	367	+	lines = append(lines, mustJSON(record))
	368	+	}
	369	+	if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
	370	+	t.Fatal(err)
	371	+	}
	372	+	spans, err := mergeProcessTraces([]processMapping{{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}})
	373	+	if tc.wantErr {
	374	+	if err == nil {
	375	+	t.Fatalf("invalid trace accepted: %+v", tc.records)
	376	+	}
	377	+	return
	378	+	}
	379	+	if err != nil \|\| len(spans) != tc.wantSpans {
	380	+	t.Fatalf("spans=%+v err=%v, want %d spans", spans, err, tc.wantSpans)
	381	+	}
	382	+	if tc.wantSpans == 1 && spans[0].Samples[0] != 10 {
	383	+	t.Fatalf("successful span=%+v, want 10ns", spans[0])
	384	+	}
	385	+	})
	386	+	}
	387	+	}

A cmd/vev-perf-harness/launcher.go +752 -0

@@ -0,0 +1,752 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"context"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"os"
	10	+	"os/exec"
	11	+	"path/filepath"
	12	+	"strconv"
	13	+	"strings"
	14	+	"sync"
	15	+	"syscall"
	16	+	"time"
	17	+
	18	+	"github.com/bnema/vev/pkg/safedir"
	19	+	"golang.org/x/sys/unix"
	20	+	)
	21	+
	22	+	type launchedProcess interface {
	23	+	Warmup([]byte) error
	24	+	Measure([]byte, func() error, func() error) error
	25	+	Close() error
	26	+	}
	27	+
	28	+	// roleCommand contains only argv accepted by app.parseArgs. Transport routing
	29	+	// metadata configures the harness's public-command seam; it is never passed as
	30	+	// an unrecognised vev flag.
	31	+	type roleCommand struct {
	32	+	Args []string
	33	+	Transport transport
	34	+	}
	35	+
	36	+	type launcher interface {
	37	+	Launch(processMapping, roleCommand) (launchedProcess, error)
	38	+	}
	39	+
	40	+	type launchedRole struct {
	41	+	mapping processMapping
	42	+	process launchedProcess
	43	+	}
	44	+
	45	+	// closeLaunchedRoles closes every role in reverse launch order. errors.Join
	46	+	// retains every cleanup failure in that stable order.
	47	+	func closeLaunchedRoles(processes []launchedRole) error {
	48	+	var failures []error
	49	+	for i := len(processes) - 1; i >= 0; i-- {
	50	+	if err := processes[i].process.Close(); err != nil {
	51	+	failures = append(failures, fmt.Errorf("close %s role: %w", processes[i].mapping.Role, err))
	52	+	}
	53	+	}
	54	+	return errors.Join(failures...)
	55	+	}
	56	+
	57	+	// cliLauncher is deliberately limited to executing the supplied public vev
	58	+	// binary. Scenario orchestration never reaches into an in-process daemon.
	59	+	type cliLauncher struct {
	60	+	bin string
	61	+	netemFactory udpNetemFactory
	62	+	mu sync.Mutex
	63	+	peers map[string]peerRoute
	64	+	runtimes map[string]string
	65	+	}
	66	+
	67	+	type peerRoute struct {
	68	+	mapping processMapping
	69	+	command roleCommand
	70	+	pidPath string
	71	+	netem udpNetem
	72	+	}
	73	+
	74	+	type preparedPeer struct {
	75	+	route peerRoute
	76	+	cleanupRuntime func() error
	77	+	}
	78	+
	79	+	func (p preparedPeer) Warmup([]byte) error { return nil }
	80	+
	81	+	func (p preparedPeer) Measure([]byte, func() error, func() error) error { return nil }
	82	+
	83	+	func (p preparedPeer) Close() error {
	84	+	var errs []error
	85	+	if p.route.pidPath != "" {
	86	+	b, err := os.ReadFile(p.route.pidPath)
	87	+	if err == nil {
	88	+	pid, parseErr := strconv.Atoi(strings.TrimSpace(string(b)))
	89	+	if parseErr != nil \|\| pid <= 0 {
	90	+	errs = append(errs, fmt.Errorf("invalid owned peer pid: %q", b))
	91	+	} else if killErr := syscall.Kill(pid, syscall.SIGTERM); killErr != nil && !errors.Is(killErr, syscall.ESRCH) {
	92	+	errs = append(errs, killErr)
	93	+	}
	94	+	} else if !errors.Is(err, os.ErrNotExist) {
	95	+	errs = append(errs, err)
	96	+	}
	97	+	}
	98	+	if p.route.netem != nil {
	99	+	if err := p.route.netem.Close(); err != nil {
	100	+	errs = append(errs, err)
	101	+	}
	102	+	}
	103	+	if p.cleanupRuntime != nil {
	104	+	if err := p.cleanupRuntime(); err != nil {
	105	+	errs = append(errs, err)
	106	+	}
	107	+	}
	108	+	return errors.Join(errs...)
	109	+	}
	110	+
	111	+	type terminalOutput interface {
	112	+	io.Writer
	113	+	Sync() error
	114	+	Close() error
	115	+	}
	116	+
	117	+	type cliProcess struct {
	118	+	cmd *exec.Cmd
	119	+	pty io.ReadWriteCloser
	120	+	output terminalOutput
	121	+	chunks chan []byte
	122	+	done chan struct{}
	123	+	closed sync.Once
	124	+	waitErr chan error
	125	+	waitTimeout func() <-chan time.Time
	126	+	forceCleanup func()
	127	+	forceKill func()
	128	+	shutdown func() error
	129	+	cleanupRuntime func() error
	130	+	terminalReady bool
	131	+	readyPath string
	132	+	}
	133	+
	134	+	func (l *cliLauncher) Launch(m processMapping, role roleCommand) (launchedProcess, error) {
	135	+	if len(role.Args) == 0 {
	136	+	return nil, errors.New("public CLI role has no arguments")
	137	+	}
	138	+	if info, err := os.Stat(l.bin); err != nil {
	139	+	return nil, err
	140	+	} else if info.IsDir() {
	141	+	return nil, fmt.Errorf("public CLI binary %q is a directory", l.bin)
	142	+	}
	143	+	if m.Role == "ssh_stdio_peer" \|\| m.Role == "udp_peer" {
	144	+	return l.preparePeer(m, role)
	145	+	}
	146	+	cmd := exec.Command(l.bin, role.Args...)
	147	+	runDir := filepath.Dir(m.TracePath)
	148	+	runtimeDir, err := l.runtimeDir(runDir)
	149	+	if err != nil {
	150	+	return nil, err
	151	+	}
	152	+	// All role inputs are absolute paths resolved at harness startup. Do not set
	153	+	// a role working directory: daemon-owned subprocess cleanup may otherwise
	154	+	// treat the evidence directory as its working tree and remove a preallocated
	155	+	// trace while a later repetition is being merged.
	156	+	cmd.Env = append(withoutEnv(os.Environ(), "VEV", "VEV_PERF_TRACE", "VEV_PERF_PROCESS_ID", "VEV_PERF_SCENARIO", "VEV_PERF_RUN", "VEV_REMOTE_TRANSPORT", "XDG_RUNTIME_DIR", "XDG_STATE_HOME"), traceEnvironment(m)...)
	157	+	cmd.Env = append(cmd.Env, "XDG_RUNTIME_DIR="+runtimeDir,
	158	+	"XDG_STATE_HOME="+filepath.Join(runDir, "state"), "TERM=xterm-256color")
	159	+	if m.Role == "client" {
	160	+	l.mu.Lock()
	161	+	peer, routed := l.peers[runDir]
	162	+	l.mu.Unlock()
	163	+	if routed {
	164	+	mode, err := remoteMode(peer.command.Transport)
	165	+	if err != nil {
	166	+	return nil, err
	167	+	}
	168	+	cmd.Env = append(cmd.Env, "PATH="+runDir+":"+os.Getenv("PATH"), "VEV_REMOTE_TRANSPORT="+mode)
	169	+	}
	170	+	}
	171	+	if err := safedir.EnsurePrivate(filepath.Join(runDir, "state")); err != nil {
	172	+	return nil, err
	173	+	}
	174	+	p := &cliProcess{cmd: cmd, waitErr: make(chan error, 1), readyPath: filepath.Join(runtimeDir, "vev", "daemon.sock")}
	175	+	if m.Role == "daemon" {
	176	+	p.cleanupRuntime = func() error { return l.releaseRuntime(runDir) }
	177	+	}
	178	+	if m.Role == "client" {
	179	+	master, slave, err := openPTY()
	180	+	if err != nil {
	181	+	return nil, err
	182	+	}
	183	+	output, err := os.OpenFile(filepath.Join(filepath.Dir(m.TracePath), "terminal-output"), os.O_WRONLY\|os.O_CREATE\|os.O_APPEND, 0o600)
	184	+	if err != nil {
	185	+	_ = master.Close()
	186	+	_ = slave.Close()
	187	+	return nil, err
	188	+	}
	189	+	cmd.Stdin, cmd.Stdout, cmd.Stderr = slave, slave, slave
	190	+	// Ctty is the child descriptor number after Start remaps slave to stdin.
	191	+	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true, Setctty: true, Ctty: 0}
	192	+	if err := cmd.Start(); err != nil {
	193	+	_ = output.Close()
	194	+	_ = master.Close()
	195	+	_ = slave.Close()
	196	+	if p.cleanupRuntime != nil {
	197	+	_ = p.cleanupRuntime()
	198	+	}
	199	+	return nil, err
	200	+	}
	201	+	_ = slave.Close()
	202	+	p.pty, p.output, p.chunks, p.done = master, output, make(chan []byte, 32), make(chan struct{})
	203	+	go p.copyTerminal()
	204	+	} else {
	205	+	// Every launched role owns a dedicated process group, so the bounded
	206	+	// cleanup path never signals the harness/test runner's group.
	207	+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	208	+	devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0)
	209	+	if err != nil {
	210	+	return nil, err
	211	+	}
	212	+	cmd.Stdin, cmd.Stdout, cmd.Stderr = devNull, devNull, devNull
	213	+	if err := cmd.Start(); err != nil {
	214	+	_ = devNull.Close()
	215	+	if p.cleanupRuntime != nil {
	216	+	_ = p.cleanupRuntime()
	217	+	}
	218	+	return nil, err
	219	+	}
	220	+	_ = devNull.Close()
	221	+	}
	222	+	if m.Role == "daemon" {
	223	+	// Stop the foreground daemon through its public CLI before its bounded
	224	+	// process-group fallback. This lets blocked carriage operations close and
	225	+	// serialize their failed end marks instead of being cut off by SIGKILL.
	226	+	p.shutdown = func() error {
	227	+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	228	+	defer cancel()
	229	+	shutdown := exec.CommandContext(ctx, l.bin, "kill", "--daemon")
	230	+	shutdown.Env = withoutEnv(cmd.Env, "VEV_PERF_TRACE", "VEV_PERF_PROCESS_ID", "VEV_PERF_SCENARIO", "VEV_PERF_RUN")
	231	+	return shutdown.Run()
	232	+	}
	233	+	}
	234	+	go func() { p.waitErr <- cmd.Wait() }()
	235	+	return p, nil
	236	+	}
	237	+
	238	+	// runtimeDir returns one short, private runtime directory shared by the roles
	239	+	// of one run. Keeping the Unix socket outside deeply nested evidence output
	240	+	// avoids the platform socket-path limit without weakening run isolation.
	241	+	func (l *cliLauncher) runtimeDir(runDir string) (string, error) {
	242	+	l.mu.Lock()
	243	+	defer l.mu.Unlock()
	244	+	if path := l.runtimes[runDir]; path != "" {
	245	+	return path, nil
	246	+	}
	247	+	path, err := os.MkdirTemp("", "vev-harness-runtime-")
	248	+	if err != nil {
	249	+	return "", err
	250	+	}
	251	+	if err := safedir.EnsurePrivate(path); err != nil {
	252	+	_ = os.RemoveAll(path)
	253	+	return "", err
	254	+	}
	255	+	if l.runtimes == nil {
	256	+	l.runtimes = make(map[string]string)
	257	+	}
	258	+	l.runtimes[runDir] = path
	259	+	return path, nil
	260	+	}
	261	+
	262	+	func (l *cliLauncher) releaseRuntime(runDir string) error {
	263	+	l.mu.Lock()
	264	+	path := l.runtimes[runDir]
	265	+	delete(l.runtimes, runDir)
	266	+	l.mu.Unlock()
	267	+	if path == "" {
	268	+	return nil
	269	+	}
	270	+	return os.RemoveAll(path)
	271	+	}
	272	+
	273	+	// preparePeer installs a per-run ssh command seam. The ordinary public client
	274	+	// still executes its documented remote attach command; its ssh child invokes
	275	+	// only the parsed public _stdio or _udp-proxy command. This makes the peer
	276	+	// carrying client traffic the declared, exclusively traced role rather than a
	277	+	// separately launched but unused process.
	278	+	func (l *cliLauncher) preparePeer(m processMapping, role roleCommand) (launchedProcess, error) {
	279	+	runDir := filepath.Dir(m.TracePath)
	280	+	l.mu.Lock()
	281	+	runtimeDir := l.runtimes[runDir]
	282	+	l.mu.Unlock()
	283	+	peerOwnsRuntime := runtimeDir == ""
	284	+	if peerOwnsRuntime {
	285	+	var err error
	286	+	runtimeDir, err = l.runtimeDir(runDir)
	287	+	if err != nil {
	288	+	return nil, err
	289	+	}
	290	+	}
	291	+	if err := safedir.EnsurePrivate(runDir); err != nil {
	292	+	return nil, err
	293	+	}
	294	+	if err := safedir.EnsurePrivate(runtimeDir); err != nil {
	295	+	return nil, err
	296	+	}
	297	+	if err := safedir.EnsurePrivate(filepath.Join(runDir, "state")); err != nil {
	298	+	return nil, err
	299	+	}
	300	+	if len(role.Args) != 2 \|\| (role.Args[0] != "_stdio" && role.Args[0] != "_udp-proxy") {
	301	+	return nil, fmt.Errorf("unsupported public peer command %q", role.Args)
	302	+	}
	303	+	route := peerRoute{mapping: m, command: role}
	304	+	if m.Role == "udp_peer" {
	305	+	route.pidPath = filepath.Join(runDir, "udp-peer.pid")
	306	+	factory := l.netemFactory
	307	+	if factory == nil {
	308	+	factory = newUDPNetem
	309	+	}
	310	+	netem, err := factory(udpNetemConfig{
	311	+	RTT: time.Duration(role.Transport.RTTMS) * time.Millisecond,
	312	+	LossPercent: role.Transport.LossPercent,
	313	+	TargetPath: filepath.Join(runDir, "udp-peer.target"),
	314	+	})
	315	+	if err != nil {
	316	+	return nil, fmt.Errorf("start harness UDP netem: %w", err)
	317	+	}
	318	+	route.netem = netem
	319	+	}
	320	+	shim := filepath.Join(runDir, "ssh")
	321	+	ready := filepath.Join(runDir, "udp-peer.ready")
	322	+	target := filepath.Join(runDir, "udp-peer.target")
	323	+	netemPort := 0
	324	+	if route.netem != nil {
	325	+	netemPort = route.netem.Port()
	326	+	}
	327	+	stderr := filepath.Join(runDir, "udp-peer.stderr")
	328	+	// The bootstrap protocol requires the SSH command to exit after forwarding
	329	+	// the readiness line. The long-lived proxy is owned by pidPath and receives
	330	+	// the role's unique trace identity, never the bootstrap/client identity.
	331	+	body := fmt.Sprintf(`#!/bin/sh
	332	+	set -eu
	333	+	case "$*" in
	334	+	"_stdio")
	335	+	exec env VEV_PERF_TRACE=%[1]q VEV_PERF_PROCESS_ID=%[2]q VEV_PERF_SCENARIO=%[12]q VEV_PERF_RUN=%[13]d XDG_RUNTIME_DIR=%[3]q XDG_STATE_HOME=%[4]q TERM=xterm-256color %[5]q _stdio %[6]q
	336	+	;;
	337	+	"_udp-bootstrap")
	338	+	rm -f %[7]q %[8]q %[9]q
	339	+	env VEV_PERF_TRACE=%[1]q VEV_PERF_PROCESS_ID=%[2]q VEV_PERF_SCENARIO=%[12]q VEV_PERF_RUN=%[13]d XDG_RUNTIME_DIR=%[3]q XDG_STATE_HOME=%[4]q TERM=xterm-256color %[5]q _udp-proxy %[6]q >%[7]q 2>%[8]q &
	340	+	peer=$!
	341	+	printf '%%s\n' "$peer" > %[11]q
	342	+	i=0
	343	+	while [ ! -s %[7]q ] && [ "$i" -lt 1000 ]; do sleep 0.01; i=$((i+1)); done
	344	+	test -s %[7]q
	345	+	line=$(head -n 1 %[7]q)
	346	+	printf '%%s\n' "$line" > %[9]q
	347	+	set -- $line
	348	+	test "$1" = VEV-UDP
	349	+	test -n "$3"
	350	+	printf 'VEV-UDP %%s %%s\n' %[10]d "$3"
	351	+	exit 0
	352	+	;;
	353	+	*) echo 'vev harness ssh seam rejected non-vev command' >&2; exit 64 ;;
	354	+	esac
	355	+	`, m.TracePath, m.ProcessID, runtimeDir, filepath.Join(runDir, "state"), l.bin, role.Args[1], ready, stderr, target, netemPort, route.pidPath, m.Scenario, m.Run)
	356	+	if err := os.WriteFile(shim, []byte(body), 0o700); err != nil {
	357	+	if route.netem != nil {
	358	+	_ = route.netem.Close()
	359	+	}
	360	+	return nil, err
	361	+	}
	362	+	l.mu.Lock()
	363	+	if l.peers == nil {
	364	+	l.peers = make(map[string]peerRoute)
	365	+	}
	366	+	if _, exists := l.peers[runDir]; exists {
	367	+	l.mu.Unlock()
	368	+	if route.netem != nil {
	369	+	_ = route.netem.Close()
	370	+	}
	371	+	return nil, fmt.Errorf("duplicate transport peer for %s", runDir)
	372	+	}
	373	+	l.peers[runDir] = route
	374	+	l.mu.Unlock()
	375	+	peer := preparedPeer{route: route}
	376	+	if peerOwnsRuntime {
	377	+	peer.cleanupRuntime = func() error { return l.releaseRuntime(runDir) }
	378	+	}
	379	+	return peer, nil
	380	+	}
	381	+
	382	+	func remoteMode(t transport) (string, error) {
	383	+	switch t.Kind {
	384	+	case "ssh_stdio":
	385	+	return "stdio", nil
	386	+	case "udp":
	387	+	return "udp", nil
	388	+	default:
	389	+	return "", fmt.Errorf("transport %q cannot route through a remote peer", t.ID)
	390	+	}
	391	+	}
	392	+
	393	+	// traceEnvironment is the complete process-local identity supplied by the
	394	+	// persisted harness processMapping. App turns Scenario/Run into production
	395	+	// mark correlation; sequence/request/epoch remain process-local operation IDs.
	396	+	func traceEnvironment(m processMapping) []string {
	397	+	return []string{
	398	+	"VEV_PERF_TRACE=" + m.TracePath,
	399	+	"VEV_PERF_PROCESS_ID=" + m.ProcessID,
	400	+	"VEV_PERF_SCENARIO=" + m.Scenario,
	401	+	"VEV_PERF_RUN=" + strconv.Itoa(m.Run),
	402	+	}
	403	+	}
	404	+
	405	+	func withoutEnv(env []string, names ...string) []string {
	406	+	remove := make(map[string]bool, len(names))
	407	+	for _, name := range names {
	408	+	remove[name] = true
	409	+	}
	410	+	out := make([]string, 0, len(env))
	411	+	for _, entry := range env {
	412	+	name, _, _ := strings.Cut(entry, "=")
	413	+	if !remove[name] {
	414	+	out = append(out, entry)
	415	+	}
	416	+	}
	417	+	return out
	418	+	}
	419	+
	420	+	func (p *cliProcess) copyTerminal() {
	421	+	defer close(p.chunks)
	422	+	buf := make([]byte, 4096)
	423	+	for {
	424	+	n, err := p.pty.Read(buf)
	425	+	if n > 0 {
	426	+	chunk := append([]byte(nil), buf[:n]...)
	427	+	if _, writeErr := p.output.Write(chunk); writeErr != nil {
	428	+	return
	429	+	}
	430	+	select {
	431	+	case p.chunks <- chunk:
	432	+	case <-p.done:
	433	+	return
	434	+	}
	435	+	}
	436	+	if err != nil {
	437	+	return
	438	+	}
	439	+	}
	440	+	}
	441	+
	442	+	// WaitReady confirms the explicitly launched public daemon owns its unique
	443	+	// socket before a client is launched. Without this barrier, the normal client
	444	+	// auto-spawn behavior can create an unmanifested daemon process during startup.
	445	+	func (p *cliProcess) WaitReady() error {
	446	+	if p.readyPath == "" {
	447	+	return errors.New("daemon readiness path is empty")
	448	+	}
	449	+	ticker := time.NewTicker(10 * time.Millisecond)
	450	+	defer ticker.Stop()
	451	+	timer := time.NewTimer(10 * time.Second)
	452	+	defer timer.Stop()
	453	+	for {
	454	+	if info, err := os.Stat(p.readyPath); err == nil {
	455	+	if info.Mode()&os.ModeSocket == 0 {
	456	+	return fmt.Errorf("daemon readiness path %s is not a socket", p.readyPath)
	457	+	}
	458	+	return nil
	459	+	}
	460	+	select {
	461	+	case err := <-p.waitErr:
	462	+	p.waitErr <- err
	463	+	if err == nil {
	464	+	return errors.New("daemon exited before readiness")
	465	+	}
	466	+	return fmt.Errorf("daemon exited before readiness: %w", err)
	467	+	case <-ticker.C:
	468	+	case <-timer.C:
	469	+	return fmt.Errorf("timed out waiting for daemon socket %s", p.readyPath)
	470	+	}
	471	+	}
	472	+	}
	473	+
	474	+	func (p *cliProcess) Warmup(input []byte) error {
	475	+	if p.pty == nil \|\| len(input) == 0 {
	476	+	return nil
	477	+	}
	478	+	// A client cannot write application output until it has received Welcome and
	479	+	// entered raw mode. Require that observable output before submitting any
	480	+	// warmup input, so the PTY's line discipline cannot echo it locally.
	481	+	if !p.terminalReady {
	482	+	if err := p.waitForTerminalReady(); err != nil {
	483	+	return err
	484	+	}
	485	+	p.terminalReady = true
	486	+	}
	487	+	if _, err := p.pty.Write(input); err != nil {
	488	+	return err
	489	+	}
	490	+	// Do not begin measured input until the public client has rendered a real
	491	+	// shell response to warmup input. This is intentionally not a measurement.
	492	+	return p.waitForTerminalMarker(inputMarker(input), input)
	493	+	}
	494	+
	495	+	// Measure records a boundary only after the harness-owned terminal output file
	496	+	// has accepted application output and Sync has reported a successful flush.
	497	+	// The marker is injected through the real client PTY, never fabricated by the
	498	+	// harness. PTY local echo is discarded and cannot satisfy this boundary.
	499	+	func (p *cliProcess) Measure(input []byte, injected, flushed func() error) error {
	500	+	if p.pty == nil {
	501	+	return errors.New("measured process has no terminal PTY")
	502	+	}
	503	+	if _, err := p.pty.Write(input); err != nil {
	504	+	return err
	505	+	}
	506	+	// The injected callback runs only after the owned PTY accepts the bytes.
	507	+	if err := injected(); err != nil {
	508	+	return err
	509	+	}
	510	+	if err := p.waitForTerminalMarker(inputMarker(input), input); err != nil {
	511	+	return err
	512	+	}
	513	+	if err := p.output.Sync(); err != nil {
	514	+	return fmt.Errorf("flush terminal output: %w", err)
	515	+	}
	516	+	return flushed()
	517	+	}
	518	+
	519	+	// waitForTerminalReady waits for public client output produced before any
	520	+	// harness input. The client enters raw mode before its output loop can render
	521	+	// this initial prompt/state, so this is an observable readiness boundary.
	522	+	func (p *cliProcess) waitForTerminalReady() error {
	523	+	timer := time.NewTimer(10 * time.Second)
	524	+	defer timer.Stop()
	525	+	for {
	526	+	select {
	527	+	case chunk, ok := <-p.chunks:
	528	+	if !ok {
	529	+	return p.terminalClosedError("readiness output")
	530	+	}
	531	+	if len(chunk) == 0 {
	532	+	continue
	533	+	}
	534	+	if err := p.output.Sync(); err != nil {
	535	+	return fmt.Errorf("flush terminal readiness output: %w", err)
	536	+	}
	537	+	return nil
	538	+	case err := <-p.waitErr:
	539	+	// Keep the exit result available to Close, which owns reaping.
	540	+	p.waitErr <- err
	541	+	if err == nil {
	542	+	return errors.New("client exited before readiness output")
	543	+	}
	544	+	return fmt.Errorf("client exited before readiness output: %w", err)
	545	+	case <-timer.C:
	546	+	return errors.New("timed out waiting for client readiness output")
	547	+	}
	548	+	}
	549	+	}
	550	+
	551	+	func (p *cliProcess) terminalClosedError(boundary string) error {
	552	+	if p.waitErr != nil {
	553	+	select {
	554	+	case err := <-p.waitErr:
	555	+	// Close owns reaping and may need the result after a failed boundary.
	556	+	p.waitErr <- err
	557	+	if err != nil {
	558	+	return fmt.Errorf("client terminal closed before %s: %w", boundary, err)
	559	+	}
	560	+	case <-time.After(10 * time.Millisecond):
	561	+	}
	562	+	}
	563	+	return fmt.Errorf("client terminal closed before %s", boundary)
	564	+	}
	565	+
	566	+	func (p *cliProcess) waitForTerminalMarker(marker, input []byte) error {
	567	+	var received []byte
	568	+	echo := newPTYLocalEcho(input)
	569	+	timer := time.NewTimer(10 * time.Second)
	570	+	defer timer.Stop()
	571	+	for !bytes.Contains(received, marker) {
	572	+	select {
	573	+	case chunk, ok := <-p.chunks:
	574	+	if !ok {
	575	+	return p.terminalClosedError("application output")
	576	+	}
	577	+	received = append(received, echo.applicationOutput(chunk)...)
	578	+	case err := <-p.waitErr:
	579	+	// Keep the exit result available to Close, which owns reaping.
	580	+	p.waitErr <- err
	581	+	if err == nil {
	582	+	return errors.New("client exited before application output")
	583	+	}
	584	+	return fmt.Errorf("client exited before application output: %w", err)
	585	+	case <-timer.C:
	586	+	return errors.New("timed out waiting for application terminal output")
	587	+	}
	588	+	}
	589	+	return nil
	590	+	}
	591	+
	592	+	// ptyLocalEcho removes the one ordered input echo that a not-yet-raw slave PTY
	593	+	// can produce. It normalizes NL to the conventional ONLCR echo spelling; any
	594	+	// divergence is retained as application output. The line discipline emits the
	595	+	// echo before the child can read and respond to the same write.
	596	+	type ptyLocalEcho struct {
	597	+	expected []byte
	598	+	matched int
	599	+	done bool
	600	+	}
	601	+
	602	+	func (e *ptyLocalEcho) applicationOutput(chunk []byte) []byte {
	603	+	if e.done \|\| len(e.expected) == 0 {
	604	+	return chunk
	605	+	}
	606	+	for i, b := range chunk {
	607	+	if b != e.expected[e.matched] {
	608	+	e.done = true
	609	+	return append([]byte(nil), chunk[i:]...)
	610	+	}
	611	+	e.matched++
	612	+	if e.matched == len(e.expected) {
	613	+	e.done = true
	614	+	return append([]byte(nil), chunk[i+1:]...)
	615	+	}
	616	+	}
	617	+	return nil
	618	+	}
	619	+
	620	+	func (p *cliProcess) Close() error {
	621	+	var result error
	622	+	p.closed.Do(func() {
	623	+	if p.done != nil {
	624	+	close(p.done)
	625	+	}
	626	+	// A public terminal client owns a shell session. Ask that shell to exit
	627	+	// before closing the PTY so its transport can finish its receive span;
	628	+	// this avoids turning ordinary harness teardown into an aborted trace.
	629	+	exited := false
	630	+	if p.shutdown != nil && p.waitErr != nil {
	631	+	select {
	632	+	case <-p.waitErr:
	633	+	exited = true
	634	+	default:
	635	+	// The client may have just ended the last session and raced this
	636	+	// best-effort public shutdown. The normal bounded reaping path below
	637	+	// remains authoritative in that case.
	638	+	_ = p.shutdown()
	639	+	}
	640	+	}
	641	+	if p.pty != nil && p.waitErr != nil {
	642	+	_, _ = p.pty.Write([]byte("exit\n"))
	643	+	select {
	644	+	case <-p.waitErr:
	645	+	exited = true
	646	+	case <-p.closeDeadline():
	647	+	}
	648	+	}
	649	+	if p.pty != nil {
	650	+	_ = p.pty.Close()
	651	+	}
	652	+	if p.waitErr != nil && !exited {
	653	+	select {
	654	+	case <-p.waitErr:
	655	+	case <-p.closeDeadline():
	656	+	// Teardown is deliberately bounded: a stuck public CLI or child
	657	+	// must not hold a local harness run indefinitely.
	658	+	p.forceProcessGroupCleanup()
	659	+	select {
	660	+	case <-p.waitErr:
	661	+	case <-p.closeDeadline():
	662	+	p.forceProcessGroupKill()
	663	+	select {
	664	+	case <-p.waitErr:
	665	+	case <-p.closeDeadline():
	666	+	result = errors.New("timed out reaping public CLI process")
	667	+	}
	668	+	}
	669	+	}
	670	+	}
	671	+	if p.output != nil {
	672	+	if err := p.output.Close(); result == nil {
	673	+	result = err
	674	+	}
	675	+	}
	676	+	if p.cleanupRuntime != nil {
	677	+	if err := p.cleanupRuntime(); result == nil {
	678	+	result = err
	679	+	}
	680	+	}
	681	+	})
	682	+	return result
	683	+	}
	684	+
	685	+	func (p *cliProcess) closeDeadline() <-chan time.Time {
	686	+	if p.waitTimeout != nil {
	687	+	return p.waitTimeout()
	688	+	}
	689	+	if p.shutdown != nil {
	690	+	// A public kill asks the daemon to finish bounded detached notices and
	691	+	// close blocked carriage before it exits; do not turn that graceful
	692	+	// protocol into a SIGKILL at the two-second client deadline.
	693	+	return time.After(5 * time.Second)
	694	+	}
	695	+	return time.After(2 * time.Second)
	696	+	}
	697	+
	698	+	func (p *cliProcess) forceProcessGroupCleanup() {
	699	+	if p.forceCleanup != nil {
	700	+	p.forceCleanup()
	701	+	return
	702	+	}
	703	+	if p.cmd != nil && p.cmd.Process != nil {
	704	+	// The client is a session/process-group leader; forced cleanup must
	705	+	// include its ssh seam/_stdio descendants.
	706	+	_ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGTERM)
	707	+	}
	708	+	}
	709	+
	710	+	func (p *cliProcess) forceProcessGroupKill() {
	711	+	if p.forceKill != nil {
	712	+	p.forceKill()
	713	+	return
	714	+	}
	715	+	if p.cmd != nil && p.cmd.Process != nil {
	716	+	_ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL)
	717	+	}
	718	+	}
	719	+
	720	+	// openPTY is the small Linux PTY boundary needed to drive the public terminal
	721	+	// client. No daemon or adapter implementation is imported by this command.
	722	+	func openPTY() (os.File, os.File, error) {
	723	+	masterFD, err := unix.Open("/dev/ptmx", unix.O_RDWR\|unix.O_NOCTTY, 0)
	724	+	if err != nil {
	725	+	return nil, nil, err
	726	+	}
	727	+	closeMaster := func(err error) (os.File, os.File, error) { _ = unix.Close(masterFD); return nil, nil, err }
	728	+	number, err := unix.IoctlGetInt(masterFD, unix.TIOCGPTN)
	729	+	if err != nil {
	730	+	return closeMaster(err)
	731	+	}
	732	+	if err := unix.IoctlSetPointerInt(masterFD, unix.TIOCSPTLCK, 0); err != nil {
	733	+	return closeMaster(err)
	734	+	}
	735	+	slaveFD, err := unix.Open(fmt.Sprintf("/dev/pts/%d", number), unix.O_RDWR\|unix.O_NOCTTY, 0)
	736	+	if err != nil {
	737	+	return closeMaster(err)
	738	+	}
	739	+	return os.NewFile(uintptr(masterFD), "vev-client-master"), os.NewFile(uintptr(slaveFD), "vev-client-slave"), nil
	740	+	}
	741	+
	742	+	func inputMarker(input []byte) []byte {
	743	+	start := bytes.Index(input, []byte("__VEV_HARNESS_"))
	744	+	if start < 0 {
	745	+	return input
	746	+	}
	747	+	end := bytes.Index(input[start:], []byte(`\n`))
	748	+	if end < 0 {
	749	+	return input[start:]
	750	+	}
	751	+	return input[start : start+end]
	752	+	}

A cmd/vev-perf-harness/launcher_test.go +296 -0

@@ -0,0 +1,296 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"fmt"
	6	+	"net"
	7	+	"os"
	8	+	"os/exec"
	9	+	"path/filepath"
	10	+	"strings"
	11	+	"testing"
	12	+	"time"
	13	+	)
	14	+
	15	+	func TestCLIProcessCloseIsBoundedWhenPublicRoleDoesNotExit(t *testing.T) {
	16	+	deadline := make(chan time.Time)
	17	+	close(deadline)
	18	+	var term, kill int
	19	+	p := &cliProcess{
	20	+	waitErr: make(chan error),
	21	+	waitTimeout: func() <-chan time.Time { return deadline },
	22	+	forceCleanup: func() { term++ },
	23	+	forceKill: func() { kill++ },
	24	+	}
	25	+	if err := p.Close(); err == nil {
	26	+	t.Fatal("stuck public role close succeeded")
	27	+	}
	28	+	if term != 1 \|\| kill != 1 {
	29	+	t.Fatalf("TERM/KILL calls = %d/%d, want 1/1", term, kill)
	30	+	}
	31	+	}
	32	+
	33	+	func TestCLIProcessWaitReadyRequiresDaemonSocket(t *testing.T) {
	34	+	path := filepath.Join(t.TempDir(), "daemon.sock")
	35	+	listener, err := net.Listen("unix", path)
	36	+	if err != nil {
	37	+	t.Fatal(err)
	38	+	}
	39	+	defer listener.Close()
	40	+	if err := (&cliProcess{readyPath: path, waitErr: make(chan error, 1)}).WaitReady(); err != nil {
	41	+	t.Fatal(err)
	42	+	}
	43	+	}
	44	+
	45	+	func TestCLILauncherIsolatesRoleXDGEnvironmentAcrossRepetitions(t *testing.T) {
	46	+	captureDir := t.TempDir()
	47	+	t.Setenv("CAPTURE_DIR", captureDir)
	48	+	t.Setenv("XDG_RUNTIME_DIR", filepath.Join(t.TempDir(), "inherited-runtime"))
	49	+	t.Setenv("XDG_STATE_HOME", filepath.Join(t.TempDir(), "inherited-state"))
	50	+
	51	+	bin := filepath.Join(t.TempDir(), "capture-env")
	52	+	if err := os.WriteFile(bin, []byte("#!/bin/sh\nenv > \"$CAPTURE_DIR/$VEV_PERF_PROCESS_ID\"\n"), 0o700); err != nil {
	53	+	t.Fatal(err)
	54	+	}
	55	+	launcher := &cliLauncher{bin: bin}
	56	+	for run := 1; run <= 3; run++ {
	57	+	runDir := filepath.Join(t.TempDir(), fmt.Sprintf("run-%d", run))
	58	+	if err := os.MkdirAll(runDir, 0o700); err != nil {
	59	+	t.Fatal(err)
	60	+	}
	61	+	roles := []string{"daemon", "client"}
	62	+	for _, role := range roles {
	63	+	id := fmt.Sprintf("run-%d-%s", run, role)
	64	+	p, err := launcher.Launch(processMapping{
	65	+	Role: role, TracePath: filepath.Join(runDir, role+".jsonl"), ProcessID: id,
	66	+	}, roleCommand{Args: []string{"test"}})
	67	+	if err != nil {
	68	+	t.Fatalf("run %d %s launch: %v", run, role, err)
	69	+	}
	70	+	runtimeDir := launcher.runtimes[runDir]
	71	+	if err := p.Close(); err != nil {
	72	+	t.Fatalf("run %d %s close: %v", run, role, err)
	73	+	}
	74	+
	75	+	env, err := os.ReadFile(filepath.Join(captureDir, id))
	76	+	if err != nil {
	77	+	t.Fatalf("run %d %s capture: %v", run, role, err)
	78	+	}
	79	+	for _, want := range []string{
	80	+	"XDG_RUNTIME_DIR=" + runtimeDir,
	81	+	"XDG_STATE_HOME=" + filepath.Join(runDir, "state"),
	82	+	} {
	83	+	name := strings.SplitN(want, "=", 2)[0]
	84	+	if count := bytes.Count(env, []byte(name+"=")); count != 1 {
	85	+	t.Fatalf("run %d %s %s entries = %d, want exactly one; env:\n%s", run, role, name, count, env)
	86	+	}
	87	+	if !bytes.Contains(env, []byte(want+"\n")) {
	88	+	t.Fatalf("run %d %s missing %q; env:\n%s", run, role, want, env)
	89	+	}
	90	+	}
	91	+	}
	92	+	if err := launcher.releaseRuntime(runDir); err != nil {
	93	+	t.Fatalf("run %d release runtime: %v", run, err)
	94	+	}
	95	+	}
	96	+	}
	97	+
	98	+	func TestCLIProcessCloseUsesPublicDaemonShutdown(t *testing.T) {
	99	+	wait := make(chan error, 1)
	100	+	wait <- nil
	101	+	var shutdown int
	102	+	p := &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; return nil }}
	103	+	if err := p.Close(); err != nil {
	104	+	t.Fatal(err)
	105	+	}
	106	+	if shutdown != 0 {
	107	+	t.Fatalf("shutdown calls = %d, want 0 for an exited daemon", shutdown)
	108	+	}
	109	+
	110	+	wait = make(chan error, 1)
	111	+	p = &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; wait <- nil; return nil }}
	112	+	if err := p.Close(); err != nil {
	113	+	t.Fatal(err)
	114	+	}
	115	+	if shutdown != 1 {
	116	+	t.Fatalf("shutdown calls = %d, want 1", shutdown)
	117	+	}
	118	+	}
	119	+
	120	+	func TestHarnessCanonicalLocalSmokeRealRoles(t *testing.T) {
	121	+	if testing.Short() {
	122	+	t.Skip("public CLI smoke")
	123	+	}
	124	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	125	+	if err != nil {
	126	+	t.Fatal(err)
	127	+	}
	128	+	bin := filepath.Join(t.TempDir(), "vev")
	129	+	build := exec.Command("go", "build", "-o", bin, "./")
	130	+	build.Dir = root
	131	+	if output, err := build.CombinedOutput(); err != nil {
	132	+	t.Fatalf("build public CLI: %v\n%s", err, output)
	133	+	}
	134	+	m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
	135	+	if err != nil {
	136	+	t.Fatal(err)
	137	+	}
	138	+	if err := validateManifest(m); err != nil {
	139	+	t.Fatal(err)
	140	+	}
	141	+	var local scenario
	142	+	for _, s := range m.Scenarios {
	143	+	if s.Topology == "1x4" && s.Workload == "idle" && s.Transport == "local" {
	144	+	local = s
	145	+	break
	146	+	}
	147	+	}
	148	+	if local.ID == "" {
	149	+	t.Fatal("canonical local scenario missing")
	150	+	}
	151	+	dir := t.TempDir()
	152	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	153	+	if err != nil {
	154	+	t.Fatal(err)
	155	+	}
	156	+	defer raw.Close()
	157	+	h := defaultHarness()
	158	+	h.clock = &fakeClock{} // injected duration keeps the public 30s contract bounded in test
	159	+	h.launcher = &cliLauncher{bin: bin}
	160	+	result, err := h.runOne(options{vevBin: bin, out: dir, duration: minimumDuration}, m, local, 1, raw)
	161	+	if err != nil {
	162	+	t.Fatal(err)
	163	+	}
	164	+	if result.Samples < minimumInIntervalEventSamples \|\| len(result.Processes) != 2 {
	165	+	t.Fatalf("canonical local smoke result=%+v", result)
	166	+	}
	167	+	}
	168	+
	169	+	func TestTraceEnvironmentMatchesProcessMapping(t *testing.T) {
	170	+	mapping := processMapping{TracePath: "/tmp/trace.jsonl", ProcessID: "p-01", Scenario: "1x4-idle-local", Run: 3}
	171	+	got := strings.Join(traceEnvironment(mapping), "\n")
	172	+	for _, want := range []string{"VEV_PERF_TRACE=/tmp/trace.jsonl", "VEV_PERF_PROCESS_ID=p-01", "VEV_PERF_SCENARIO=1x4-idle-local", "VEV_PERF_RUN=3"} {
	173	+	if !strings.Contains(got, want) {
	174	+	t.Errorf("trace environment missing %q: %s", want, got)
	175	+	}
	176	+	}
	177	+	}
	178	+
	179	+	func TestCLIProcessCloseGracefulDetachCompletesSpanBeforeForcedCleanup(t *testing.T) {
	180	+	var order []string
	181	+	waitErr := make(chan error, 1)
	182	+	timeout := make(chan time.Time)
	183	+	p := &cliProcess{
	184	+	pty: &orderedPTY{
	185	+	write: func(input []byte) {
	186	+	if string(input) != "exit\n" {
	187	+	t.Fatalf("graceful exit input=%q", input)
	188	+	}
	189	+	order = append(order, "exit_written")
	190	+	// The graceful client exit has closed its transport and emitted
	191	+	// its adapter receive-end mark before cmd.Wait completes.
	192	+	order = append(order, "adapter_receive_end")
	193	+	waitErr <- nil
	194	+	},
	195	+	close: func() { order = append(order, "pty_closed") },
	196	+	},
	197	+	output: &orderedOutput{close: func() { order = append(order, "output_closed") }},
	198	+	waitErr: waitErr,
	199	+	waitTimeout: func() <-chan time.Time { return timeout },
	200	+	forceCleanup: func() {
	201	+	order = append(order, "forced_process_group_cleanup")
	202	+	},
	203	+	}
	204	+
	205	+	if err := p.Close(); err != nil {
	206	+	t.Fatal(err)
	207	+	}
	208	+	if !equalStrings(order, []string{"exit_written", "adapter_receive_end", "pty_closed", "output_closed"}) {
	209	+	t.Fatalf("cleanup order=%q", order)
	210	+	}
	211	+	}
	212	+
	213	+	func TestCLIProcessWarmupWaitsForDelayedTerminalReadiness(t *testing.T) {
	214	+	allowWrite := make(chan struct{})
	215	+	pty, output := &stagedPTY{allow: allowWrite, writes: make(chan []byte, 1)}, &fakeOutput{}
	216	+	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte)}
	217	+	input := workloadInput(scenario{ID: "s"}, 1, "warmup")
	218	+	done := make(chan error, 1)
	219	+	go func() { done <- p.Warmup(input) }()
	220	+
	221	+	// This is public client output from the initial application prompt/state,
	222	+	// emitted only after the client has entered raw mode. A premature PTY write
	223	+	// blocks on allowWrite and therefore cannot consume this delayed readiness.
	224	+	go func() {
	225	+	p.chunks <- []byte("shell prompt$ ")
	226	+	close(allowWrite)
	227	+	}()
	228	+	select {
	229	+	case got := <-pty.writes:
	230	+	if !bytes.Equal(got, input) {
	231	+	t.Fatalf("warmup write=%q want %q", got, input)
	232	+	}
	233	+	case <-time.After(time.Second):
	234	+	t.Fatal("warmup did not wait for and then proceed from client readiness")
	235	+	}
	236	+	go func() { p.chunks <- append([]byte("application "), inputMarker(input)...); close(p.chunks) }()
	237	+	if err := <-done; err != nil {
	238	+	t.Fatal(err)
	239	+	}
	240	+	if output.syncs != 1 {
	241	+	t.Fatalf("readiness output was not flushed: syncs=%d", output.syncs)
	242	+	}
	243	+	}
	244	+
	245	+	func TestCLIProcessRejectsPTYLocalEchoAsFlushEvidence(t *testing.T) {
	246	+	pty, output := &fakePTY{}, &fakeOutput{}
	247	+	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte, 1)}
	248	+	input := workloadInput(scenario{ID: "s"}, 1, "measured-1")
	249	+	p.chunks <- newPTYLocalEcho(input).expected
	250	+	close(p.chunks)
	251	+	flushed := false
	252	+	if err := p.Measure(input, func() error { return nil }, func() error { flushed = true; return nil }); err == nil {
	253	+	t.Fatal("local PTY echo satisfied terminal flush boundary")
	254	+	}
	255	+	if flushed \|\| output.syncs != 0 {
	256	+	t.Fatalf("local echo stamped a flush: flushed=%t syncs=%d", flushed, output.syncs)
	257	+	}
	258	+	}
	259	+
	260	+	func TestCLIProcessPairsApplicationOutputWithSuccessfulFlush(t *testing.T) {
	261	+	pty, output := &fakePTY{}, &fakeOutput{}
	262	+	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte, 1)}
	263	+	input := workloadInput(scenario{ID: "s"}, 1, "measured-1")
	264	+	echo := newPTYLocalEcho(input).expected
	265	+	p.chunks <- append(append([]byte(nil), echo...), append([]byte("application "), inputMarker(input)...)...)
	266	+	order := []string{}
	267	+	if err := p.Measure(input, func() error { order = append(order, "injected"); return nil }, func() error { order = append(order, "flushed"); return nil }); err != nil {
	268	+	t.Fatal(err)
	269	+	}
	270	+	if len(pty.writes) != 1 \|\| output.syncs != 1 \|\| !equalStrings(order, []string{"injected", "flushed"}) {
	271	+	t.Fatalf("writes=%q syncs=%d boundary order=%q", pty.writes, output.syncs, order)
	272	+	}
	273	+	}
	274	+
	275	+	func TestHarnessUsesPublicRoleCommandsAndPTYWorkloads(t *testing.T) {
	276	+	for _, tc := range []struct {
	277	+	role string
	278	+	want []string
	279	+	}{
	280	+	{"daemon", []string{"--daemon"}},
	281	+	{"client", []string{"new", "perf-s-001"}},
	282	+	{"ssh_stdio_peer", []string{"_stdio", "perf-s-001"}},
	283	+	{"udp_peer", []string{"_udp-proxy", "perf-s-001"}},
	284	+	} {
	285	+	t.Run(tc.role, func(t *testing.T) {
	286	+	got := roleArgs(scenario{ID: "s", Transport: "local"}, processMapping{Role: tc.role, Run: 1})
	287	+	if !equalStrings(got.Args, tc.want) {
	288	+	t.Fatalf("args=%q want %q", got.Args, tc.want)
	289	+	}
	290	+	})
	291	+	}
	292	+	input := string(workloadInput(scenario{ID: "s", Workload: "interactive_flood"}, 1, "measured-1"))
	293	+	if string(inputMarker([]byte(input))) != "__VEV_HARNESS_s_r1_measured-1__" \|\| !bytes.Contains([]byte(input), []byte("while [ $i -lt 128 ]")) \|\| !strings.HasSuffix(input, "printf '__VEV_HARNESS_s_r1_measured-1__\\n'\n") {
	294	+	t.Fatalf("workload is not real PTY shell input with an observable marker: %q", input)
	295	+	}
	296	+	}

A cmd/vev-perf-harness/main.go +20 -0

@@ -0,0 +1,20 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"os"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/vev/pkg/safedir"
	9	+	)
	10	+
	11	+	func main() {
	12	+	if err := run(os.Args[1:], defaultHarness()); err != nil {
	13	+	fmt.Fprintln(os.Stderr, "vev-perf-harness:", err)
	14	+	os.Exit(2)
	15	+	}
	16	+	}
	17	+
	18	+	func defaultHarness() *harness {
	19	+	return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, createRunDir: createExclusiveRunDir, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}
	20	+	}

A cmd/vev-perf-harness/model.go +132 -0

@@ -0,0 +1,132 @@
	1	+	package main
	2	+
	3	+	import "time"
	4	+
	5	+	const (
	6	+	minimumDuration = 30 * time.Second
	7	+	minimumRepetitions = 10
	8	+	minimumInIntervalEventSamples = 10
	9	+	measuredEventCadence = time.Second
	10	+	)
	11	+
	12	+	type options struct {
	13	+	vevBin, manifest, out string
	14	+	scenario string
	15	+	warmup, duration time.Duration
	16	+	repetitions int
	17	+	}
	18	+
	19	+	type manifest struct {
	20	+	Schema uint16 `json:"schema"`
	21	+	Topologies []topology `json:"topologies"`
	22	+	Workloads []string `json:"workloads"`
	23	+	Transports []transport `json:"transports"`
	24	+	Scenarios []scenario `json:"scenarios"`
	25	+	}
	26	+
	27	+	type topology struct {
	28	+	ID string `json:"id"`
	29	+	Geometry string `json:"geometry"`
	30	+	RowsPerPane int `json:"rows_per_pane"`
	31	+	}
	32	+
	33	+	type transport struct {
	34	+	ID string `json:"id"`
	35	+	Kind string `json:"kind"`
	36	+	RTTMS int `json:"rtt_ms,omitempty"`
	37	+	LossPercent int `json:"loss_percent,omitempty"`
	38	+	}
	39	+
	40	+	type scenario struct {
	41	+	ID string `json:"id"`
	42	+	Topology string `json:"topology"`
	43	+	Workload string `json:"workload"`
	44	+	Transport string `json:"transport"`
	45	+	Roles []string `json:"roles"`
	46	+	InapplicableReason string `json:"inapplicable_reason,omitempty"`
	47	+	}
	48	+
	49	+	type processMapping struct {
	50	+	ProcessID string `json:"process_id"`
	51	+	ClockDomain string `json:"clock_domain"`
	52	+	TracePath string `json:"trace_path"`
	53	+	Role string `json:"role"`
	54	+	Scenario string `json:"scenario"`
	55	+	Run int `json:"run"`
	56	+	Identity string `json:"identity"`
	57	+	}
	58	+
	59	+	type runManifest struct {
	60	+	Scenario string `json:"scenario"`
	61	+	Run int `json:"run"`
	62	+	Processes []processMapping `json:"processes"`
	63	+	}
	64	+
	65	+	// measuredInterval belongs solely to the harness clock domain. A paired event
	66	+	// is eligible only when both owned boundaries fall inside this interval.
	67	+	type measuredInterval struct {
	68	+	Start int64
	69	+	End int64
	70	+	}
	71	+
	72	+	type eventSample struct {
	73	+	Sequence uint64 `json:"sequence"`
	74	+	Injected int64 `json:"injected_tick"`
	75	+	Flushed int64 `json:"flushed_tick"`
	76	+	Latency int64 `json:"latency_nanos"`
	77	+	}
	78	+
	79	+	type span struct {
	80	+	Component, Name string
	81	+	Samples []int64
	82	+	}
	83	+
	84	+	type runResult struct {
	85	+	Spans []span `json:"process_local_spans"`
	86	+	Scenario string `json:"scenario"`
	87	+	Run int `json:"run"`
	88	+	Samples int `json:"samples"`
	89	+	EndToEnd []int64 `json:"end_to_end_nanos"`
	90	+	Event distribution `json:"event_end_to_end"`
	91	+	Cadence distribution `json:"event_cadence_nanos"`
	92	+	CadenceSamples []int64 `json:"-"`
	93	+	MaxGap int64 `json:"event_max_gap_nanos"`
	94	+	Processes []processMapping `json:"processes"`
	95	+	}
	96	+
	97	+	type summary struct {
	98	+	Schema uint16 `json:"schema"`
	99	+	GitSHA string `json:"git_sha"`
	100	+	Warmup string `json:"warmup"`
	101	+	Duration string `json:"duration"`
	102	+	Repetitions int `json:"repetitions"`
	103	+	EndToEnd distribution `json:"harness_end_to_end"`
	104	+	Cadence distribution `json:"harness_event_cadence_nanos"`
	105	+	MaxGap int64 `json:"harness_event_max_gap_nanos"`
	106	+	RunP50Dispersion distribution `json:"run_p50_dispersion"`
	107	+	Spans []spanSummary `json:"process_local_spans"`
	108	+	Runs int `json:"runs"`
	109	+	}
	110	+
	111	+	type distribution struct {
	112	+	Count int `json:"count"`
	113	+	P50 int64 `json:"p50"`
	114	+	P95 int64 `json:"p95"`
	115	+	P99 int64 `json:"p99"`
	116	+	Max int64 `json:"max"`
	117	+	}
	118	+
	119	+	type spanSummary struct {
	120	+	Component string `json:"component"`
	121	+	Name string `json:"name"`
	122	+	Distribution distribution `json:"distribution"`
	123	+	}
	124	+
	125	+	type clock interface {
	126	+	Now() int64
	127	+	Sleep(time.Duration)
	128	+	}
	129	+
	130	+	type systemClock struct{ start time.Time }
	131	+
	132	+	var spanPairs = []spanPair{{"capture_start", "capture_end", "capture_duration"}, {"compose_start", "compose_end", "compose_duration"}, {"diff_start", "diff_end", "diff_duration"}, {"queue_enqueued", "queue_dequeued", "queue_wait"}, {"ack_blocked_start", "ack_blocked_end", "ack_blocked_interval"}, {"emit_start", "emit_end", "emit_duration"}, {"adapter_send_start", "adapter_send_end", "adapter_send_duration"}, {"adapter_receive_start", "adapter_receive_end", "adapter_receive_duration"}}

A cmd/vev-perf-harness/options_manifest.go +221 -0

@@ -0,0 +1,221 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"errors"
	6	+	"flag"
	7	+	"fmt"
	8	+	"io"
	9	+	"os"
	10	+	"path/filepath"
	11	+	)
	12	+
	13	+	func parseOptions(args []string) (options, error) {
	14	+	var o options
	15	+	fs := flag.NewFlagSet("vev-perf-harness", flag.ContinueOnError)
	16	+	fs.SetOutput(io.Discard)
	17	+	fs.StringVar(&o.vevBin, "vev-bin", "", "public vev binary")
	18	+	fs.StringVar(&o.manifest, "manifest", "", "manifest")
	19	+	fs.StringVar(&o.out, "out", "", "output directory")
	20	+	fs.StringVar(&o.scenario, "scenario", "", "one canonical scenario ID (after full manifest validation)")
	21	+	fs.DurationVar(&o.warmup, "warmup", 0, "warmup")
	22	+	fs.DurationVar(&o.duration, "duration", 0, "measurement duration")
	23	+	fs.IntVar(&o.repetitions, "repetitions", 0, "repetitions")
	24	+	if err := fs.Parse(args); err != nil {
	25	+	return o, err
	26	+	}
	27	+	seen := map[string]bool{}
	28	+	fs.Visit(func(f *flag.Flag) { seen[f.Name] = true })
	29	+	for _, name := range []string{"vev-bin", "manifest", "out", "warmup", "duration", "repetitions"} {
	30	+	if !seen[name] {
	31	+	return o, fmt.Errorf("--%s is required", name)
	32	+	}
	33	+	}
	34	+	if o.vevBin == "" \|\| o.manifest == "" \|\| o.out == "" {
	35	+	return o, errors.New("--vev-bin, --manifest, and --out are required")
	36	+	}
	37	+	if o.warmup < 0 {
	38	+	return o, errors.New("--warmup must not be negative")
	39	+	}
	40	+	if o.duration < minimumDuration {
	41	+	return o, fmt.Errorf("--duration must be at least %s", minimumDuration)
	42	+	}
	43	+	if o.repetitions < minimumRepetitions {
	44	+	return o, fmt.Errorf("--repetitions must be at least %d", minimumRepetitions)
	45	+	}
	46	+	return o, nil
	47	+	}
	48	+
	49	+	// resolvePathOptions captures the harness cwd once, before role launchers
	50	+	// assign their isolated working directories. All downstream filesystem access,
	51	+	// manifests, and process environments consequently refer to the same paths.
	52	+	func resolvePathOptions(o options) (options, error) {
	53	+	cwd, err := os.Getwd()
	54	+	if err != nil {
	55	+	return o, fmt.Errorf("resolve harness working directory: %w", err)
	56	+	}
	57	+	absolute := func(path string) string {
	58	+	if filepath.IsAbs(path) {
	59	+	return filepath.Clean(path)
	60	+	}
	61	+	return filepath.Clean(filepath.Join(cwd, path))
	62	+	}
	63	+	o.vevBin = absolute(o.vevBin)
	64	+	o.manifest = absolute(o.manifest)
	65	+	o.out = absolute(o.out)
	66	+	return o, nil
	67	+	}
	68	+
	69	+	func readManifest(path string) (manifest, error) {
	70	+	f, err := os.Open(path)
	71	+	if err != nil {
	72	+	return manifest{}, err
	73	+	}
	74	+	var m manifest
	75	+	if err := json.NewDecoder(f).Decode(&m); err != nil {
	76	+	return manifest{}, closeFile(f, err)
	77	+	}
	78	+	if err := f.Close(); err != nil {
	79	+	return manifest{}, err
	80	+	}
	81	+	return m, nil
	82	+	}
	83	+
	84	+	func validateManifest(m manifest) error {
	85	+	if m.Schema != 1 {
	86	+	return errors.New("unsupported manifest schema")
	87	+	}
	88	+	tops := map[string]bool{}
	89	+	for _, t := range m.Topologies {
	90	+	if t.ID == "" \|\| t.Geometry != "120x40" \|\| t.RowsPerPane != 10000 {
	91	+	return fmt.Errorf("invalid topology %q", t.ID)
	92	+	}
	93	+	tops[t.ID] = true
	94	+	}
	95	+	wantT := []string{"1x4", "4x1", "4x4", "8x1"}
	96	+	for _, v := range wantT {
	97	+	if !tops[v] {
	98	+	return fmt.Errorf("missing canonical topology %s", v)
	99	+	}
	100	+	}
	101	+	works := set(m.Workloads)
	102	+	for _, v := range []string{"idle", "active_output", "all_output", "inactive_output", "interactive_flood", "copy_search", "resize_sweep", "snapshot_output_resize", "attach_restore_tab_switch"} {
	103	+	if !works[v] {
	104	+	return fmt.Errorf("missing canonical workload %s", v)
	105	+	}
	106	+	}
	107	+	trans := map[string]bool{}
	108	+	for _, t := range m.Transports {
	109	+	if t.ID == "" {
	110	+	return errors.New("transport missing id")
	111	+	}
	112	+	if err := validateTransportFixture(t); err != nil {
	113	+	return err
	114	+	}
	115	+	trans[t.ID] = true
	116	+	}
	117	+	for _, v := range []string{"local", "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct"} {
	118	+	if !trans[v] {
	119	+	return fmt.Errorf("missing canonical transport %s", v)
	120	+	}
	121	+	}
	122	+	seen := map[string]bool{}
	123	+	covered := map[string]bool{}
	124	+	for _, s := range m.Scenarios {
	125	+	if s.ID == "" \|\| seen[s.ID] {
	126	+	return fmt.Errorf("duplicate or empty scenario id %q", s.ID)
	127	+	}
	128	+	seen[s.ID] = true
	129	+	if !tops[s.Topology] \|\| !works[s.Workload] \|\| !trans[s.Transport] {
	130	+	return fmt.Errorf("scenario %s references unknown matrix entry", s.ID)
	131	+	}
	132	+	if s.InapplicableReason == "" && len(s.Roles) == 0 {
	133	+	return fmt.Errorf("scenario %s has no process roles", s.ID)
	134	+	}
	135	+	if s.InapplicableReason != "" && len(s.Roles) != 0 {
	136	+	return fmt.Errorf("inapplicable scenario %s must not launch roles", s.ID)
	137	+	}
	138	+	if s.InapplicableReason == "" && !equalRoleSet(s.Roles, requiredRoles(s.Transport)) {
	139	+	return fmt.Errorf("scenario %s roles do not route transport %s", s.ID, s.Transport)
	140	+	}
	141	+	covered[s.Topology+"\x00"+s.Workload+"\x00"+s.Transport] = true
	142	+	}
	143	+	for topologyID := range tops {
	144	+	for workload := range works {
	145	+	for transportID := range trans {
	146	+	key := topologyID + "\x00" + workload + "\x00" + transportID
	147	+	if !covered[key] {
	148	+	return fmt.Errorf("missing scenario or inapplicable reason for %s/%s/%s", topologyID, workload, transportID)
	149	+	}
	150	+	}
	151	+	}
	152	+	}
	153	+	return nil
	154	+	}
	155	+
	156	+	// selectScenario is intentionally called only after validateManifest. A bounded
	157	+	// local evidence run therefore cannot mask an incomplete canonical matrix.
	158	+	func selectScenario(m manifest, id string) ([]scenario, error) {
	159	+	for _, s := range m.Scenarios {
	160	+	if s.ID != id {
	161	+	continue
	162	+	}
	163	+	if s.InapplicableReason != "" {
	164	+	return nil, fmt.Errorf("scenario %s is inapplicable: %s", id, s.InapplicableReason)
	165	+	}
	166	+	return []scenario{s}, nil
	167	+	}
	168	+	return nil, fmt.Errorf("unknown canonical scenario %s", id)
	169	+	}
	170	+
	171	+	func requiredRoles(transportID string) []string {
	172	+	switch transportID {
	173	+	case "local":
	174	+	return []string{"daemon", "client"}
	175	+	case "ssh_stdio":
	176	+	return []string{"daemon", "client", "ssh_stdio_peer"}
	177	+	default:
	178	+	return []string{"daemon", "client", "udp_peer"}
	179	+	}
	180	+	}
	181	+
	182	+	func equalRoleSet(got, want []string) bool {
	183	+	if len(got) != len(want) {
	184	+	return false
	185	+	}
	186	+	seen := make(map[string]bool, len(got))
	187	+	for _, role := range got {
	188	+	seen[role] = true
	189	+	}
	190	+	for _, role := range want {
	191	+	if !seen[role] {
	192	+	return false
	193	+	}
	194	+	}
	195	+	return true
	196	+	}
	197	+
	198	+	func validateTransportFixture(t transport) error {
	199	+	want := map[string]transport{
	200	+	"local": {ID: "local", Kind: "local"},
	201	+	"ssh_stdio": {ID: "ssh_stdio", Kind: "ssh_stdio"},
	202	+	"udp_baseline": {ID: "udp_baseline", Kind: "udp"},
	203	+	"udp_25ms": {ID: "udp_25ms", Kind: "udp", RTTMS: 25},
	204	+	"udp_100ms": {ID: "udp_100ms", Kind: "udp", RTTMS: 100},
	205	+	"udp_loss_0pct": {ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0},
	206	+	"udp_loss_1pct": {ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1},
	207	+	}
	208	+	w, ok := want[t.ID]
	209	+	if !ok \|\| t.Kind != w.Kind \|\| t.RTTMS != w.RTTMS \|\| t.LossPercent != w.LossPercent {
	210	+	return fmt.Errorf("invalid canonical transport fixture %+v", t)
	211	+	}
	212	+	return nil
	213	+	}
	214	+
	215	+	func set(v []string) map[string]bool {
	216	+	r := map[string]bool{}
	217	+	for _, x := range v {
	218	+	r[x] = true
	219	+	}
	220	+	return r
	221	+	}

A cmd/vev-perf-harness/options_manifest_test.go +201 -0

@@ -0,0 +1,201 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"errors"
	6	+	"os"
	7	+	"path/filepath"
	8	+	"strings"
	9	+	"testing"
	10	+	"time"
	11	+	)
	12	+
	13	+	func TestHarnessScenarioSelectionKeepsCanonicalManifestValidation(t *testing.T) {
	14	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	15	+	if err != nil {
	16	+	t.Fatal(err)
	17	+	}
	18	+	manifestPath := filepath.Join(root, "testdata", "perf", "manifest.json")
	19	+	m, err := readManifest(manifestPath)
	20	+	if err != nil {
	21	+	t.Fatal(err)
	22	+	}
	23	+	selected := m.Scenarios[0]
	24	+	h := defaultHarness()
	25	+	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
	26	+	out := filepath.Join(t.TempDir(), "out")
	27	+	if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
	28	+	t.Fatal(err)
	29	+	}
	30	+	var runs []runResult
	31	+	b, err := os.ReadFile(filepath.Join(out, "runs.json"))
	32	+	if err != nil {
	33	+	t.Fatal(err)
	34	+	}
	35	+	if err := json.Unmarshal(b, &runs); err != nil {
	36	+	t.Fatal(err)
	37	+	}
	38	+	if len(runs) != 10 {
	39	+	t.Fatalf("selected canonical runs=%d, want 10", len(runs))
	40	+	}
	41	+	for _, result := range runs {
	42	+	if result.Scenario != selected.ID {
	43	+	t.Fatalf("selected scenario = %q, want %q", result.Scenario, selected.ID)
	44	+	}
	45	+	}
	46	+	}
	47	+
	48	+	func TestHarnessScenarioSelectionRejectsIncompleteCanonicalManifest(t *testing.T) {
	49	+	m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
	50	+	if err != nil {
	51	+	t.Fatal(err)
	52	+	}
	53	+	selected := m.Scenarios[0]
	54	+	m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
	55	+	dir := t.TempDir()
	56	+	manifestPath := filepath.Join(dir, "manifest.json")
	57	+	if err := os.WriteFile(manifestPath, []byte(mustJSON(m)), 0o600); err != nil {
	58	+	t.Fatal(err)
	59	+	}
	60	+	launcher := &fakeLauncher{}
	61	+	h := defaultHarness()
	62	+	h.clock, h.launcher = &fakeClock{}, launcher
	63	+	out := filepath.Join(dir, "out")
	64	+	err = run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h)
	65	+	if err == nil \|\| !strings.Contains(err.Error(), "missing scenario or inapplicable reason") {
	66	+	t.Fatalf("incomplete manifest selection error = %v", err)
	67	+	}
	68	+	if len(launcher.mappings) != 0 {
	69	+	t.Fatalf("launched with incomplete canonical manifest: %+v", launcher.mappings)
	70	+	}
	71	+	if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
	72	+	t.Fatalf("output created before manifest validation: %v", statErr)
	73	+	}
	74	+	}
	75	+
	76	+	func TestResolvePathOptionsMakesRelativeAndAbsoluteInputsIdentical(t *testing.T) {
	77	+	cwd, err := os.Getwd()
	78	+	if err != nil {
	79	+	t.Fatal(err)
	80	+	}
	81	+	absolute := options{
	82	+	vevBin: filepath.Join(cwd, "vev"),
	83	+	manifest: filepath.Join(cwd, "testdata", "perf", "manifest.json"),
	84	+	out: filepath.Join(cwd, "testdata", "perf", "results"),
	85	+	}
	86	+	relative := options{
	87	+	vevBin: "vev",
	88	+	manifest: "testdata/perf/manifest.json",
	89	+	out: "testdata/perf/results",
	90	+	}
	91	+	gotRelative, err := resolvePathOptions(relative)
	92	+	if err != nil {
	93	+	t.Fatal(err)
	94	+	}
	95	+	gotAbsolute, err := resolvePathOptions(absolute)
	96	+	if err != nil {
	97	+	t.Fatal(err)
	98	+	}
	99	+	if gotRelative != gotAbsolute {
	100	+	t.Fatalf("relative paths resolved to %+v, absolute paths to %+v", gotRelative, gotAbsolute)
	101	+	}
	102	+	}
	103	+
	104	+	func TestHarnessFlagsRejectShortMeasurements(t *testing.T) {
	105	+	for _, args := range [][]string{
	106	+	{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "29s", "--repetitions", "10"},
	107	+	{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "9"},
	108	+	{"--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"},
	109	+	} {
	110	+	if _, err := parseOptions(args); err == nil {
	111	+	t.Fatalf("parseOptions(%v) succeeded", args)
	112	+	}
	113	+	}
	114	+	if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"}); err == nil {
	115	+	t.Fatal("missing --warmup accepted")
	116	+	}
	117	+	if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}); err != nil {
	118	+	t.Fatal(err)
	119	+	}
	120	+	}
	121	+
	122	+	func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
	123	+	m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
	124	+	if err != nil {
	125	+	t.Fatal(err)
	126	+	}
	127	+	if err := validateManifest(m); err != nil {
	128	+	t.Fatal(err)
	129	+	}
	130	+	if len(m.Scenarios) != 497 {
	131	+	t.Fatalf("scenarios=%d", len(m.Scenarios))
	132	+	}
	133	+	m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
	134	+	if err := validateManifest(m); err == nil {
	135	+	t.Fatal("missing combination accepted")
	136	+	}
	137	+	}
	138	+
	139	+	func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
	140	+	dir := t.TempDir()
	141	+	raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	142	+	if err != nil {
	143	+	t.Fatal(err)
	144	+	}
	145	+	defer raw.Close()
	146	+	launcher := &fakeLauncher{}
	147	+	h := defaultHarness()
	148	+	h.clock = &fakeClock{}
	149	+	h.launcher = launcher
	150	+	o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
	151	+	r, err := h.runOne(o, manifest{}, scenario{ID: "4x4-local", Roles: []string{"daemon", "client"}}, 1, raw)
	152	+	if err != nil {
	153	+	t.Fatal(err)
	154	+	}
	155	+	if !launcher.manifestPresent {
	156	+	t.Fatal("launched before complete run manifest")
	157	+	}
	158	+	if len(r.EndToEnd) < minimumInIntervalEventSamples \|\| r.Samples < minimumInIntervalEventSamples {
	159	+	t.Fatalf("samples=%+v", r)
	160	+	}
	161	+	seen := map[string]bool{}
	162	+	for _, m := range r.Processes {
	163	+	if m.ProcessID == "" \|\| m.ClockDomain != m.ProcessID \|\| m.TracePath == "" \|\| seen[m.TracePath] {
	164	+	t.Fatalf("bad mapping %+v", m)
	165	+	}
	166	+	seen[m.TracePath] = true
	167	+	if _, err := os.Stat(m.TracePath); err != nil {
	168	+	t.Fatal(err)
	169	+	}
	170	+	}
	171	+	b, err := os.ReadFile(filepath.Join(dir, "4x4-local-run-001", "manifest.json"))
	172	+	if err != nil {
	173	+	t.Fatal(err)
	174	+	}
	175	+	var got runManifest
	176	+	if err := json.Unmarshal(b, &got); err != nil {
	177	+	t.Fatal(err)
	178	+	}
	179	+	if len(got.Processes) != 2 {
	180	+	t.Fatalf("manifest=%+v", got)
	181	+	}
	182	+	if len(launcher.process) != 2 \|\| !launcher.process[0].closed \|\| len(launcher.process[0].warmups) != 0 \|\| len(launcher.process[1].warmups) != 1 \|\| len(launcher.process[1].measures) < minimumInIntervalEventSamples {
	183	+	t.Fatalf("process lifecycle not recorded: %+v", launcher.process)
	184	+	}
	185	+	}
	186	+
	187	+	func TestHarnessRejectsMissingManifestRoleTrace(t *testing.T) {
	188	+	dir := t.TempDir()
	189	+	missing := filepath.Join(dir, "daemon.jsonl")
	190	+	client := filepath.Join(dir, "client.jsonl")
	191	+	if err := os.WriteFile(client, nil, 0o600); err != nil {
	192	+	t.Fatal(err)
	193	+	}
	194	+	_, err := mergeProcessTraces([]processMapping{
	195	+	{ProcessID: "daemon", ClockDomain: "daemon", TracePath: missing, Role: "daemon", Scenario: "s", Run: 1},
	196	+	{ProcessID: "client", ClockDomain: "client", TracePath: client, Role: "client", Scenario: "s", Run: 1},
	197	+	})
	198	+	if !errors.Is(err, os.ErrNotExist) \|\| !strings.Contains(err.Error(), "daemon role") {
	199	+	t.Fatalf("missing daemon trace error = %v, want daemon role not-exist error", err)
	200	+	}
	201	+	}

A cmd/vev-perf-harness/runner.go +439 -0

@@ -0,0 +1,439 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"context"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"os"
	10	+	"os/exec"
	11	+	"path/filepath"
	12	+	"runtime/debug"
	13	+	"sort"
	14	+	"strings"
	15	+	"time"
	16	+
	17	+	"github.com/bnema/vev/pkg/safedir"
	18	+	)
	19	+
	20	+	type harnessMark struct {
	21	+	Scenario string `json:"scenario"`
	22	+	Run int `json:"run"`
	23	+	Sequence uint64 `json:"sequence"`
	24	+	Kind string `json:"kind"`
	25	+	Tick int64 `json:"tick"`
	26	+	Valid bool `json:"valid"`
	27	+	}
	28	+
	29	+	func (c systemClock) Now() int64 { return time.Since(c.start).Nanoseconds() }
	30	+
	31	+	func (systemClock) Sleep(d time.Duration) { time.Sleep(d) }
	32	+
	33	+	func newPTYLocalEcho(input []byte) *ptyLocalEcho {
	34	+	expected := make([]byte, 0, len(input))
	35	+	for _, b := range input {
	36	+	if b == '\n' {
	37	+	expected = append(expected, '\r')
	38	+	}
	39	+	expected = append(expected, b)
	40	+	}
	41	+	return &ptyLocalEcho{expected: expected}
	42	+	}
	43	+
	44	+	type harness struct {
	45	+	clock clock
	46	+	launcher launcher
	47	+	mkdir func(string) error
	48	+	createRunDir func(string) error
	49	+	create func(string) (*os.File, error)
	50	+	removeAll func(string) error
	51	+	gitSHA func() string
	52	+	selectScenarios func(manifest) []scenario // test-only bounded fixture seam
	53	+	}
	54	+
	55	+	// createExclusiveRunDir prevents stale runtime, socket, state, or trace files
	56	+	// from an earlier invocation being mistaken for the next repetition's evidence.
	57	+	func createExclusiveRunDir(path string) error {
	58	+	if err := os.Mkdir(path, 0o700); err != nil {
	59	+	return err
	60	+	}
	61	+	if err := safedir.EnsurePrivate(path); err != nil {
	62	+	return errors.Join(err, os.Remove(path))
	63	+	}
	64	+	return nil
	65	+	}
	66	+
	67	+	// recordedGitSHA prefers build provenance so an installed public harness does
	68	+	// not depend on its launch directory being a checkout. The git fallback keeps
	69	+	// go run evidence attributable as well.
	70	+	const gitSHACommandTimeout = time.Second
	71	+
	72	+	func recordedGitSHA() string {
	73	+	if info, ok := debug.ReadBuildInfo(); ok {
	74	+	for _, setting := range info.Settings {
	75	+	if setting.Key == "vcs.revision" && setting.Value != "" {
	76	+	return setting.Value
	77	+	}
	78	+	}
	79	+	}
	80	+	ctx, cancel := context.WithTimeout(context.Background(), gitSHACommandTimeout)
	81	+	defer cancel()
	82	+	out, err := exec.CommandContext(ctx, "git", "rev-parse", "HEAD").Output()
	83	+	if err == nil && strings.TrimSpace(string(out)) != "" {
	84	+	return strings.TrimSpace(string(out))
	85	+	}
	86	+	return "unknown"
	87	+	}
	88	+
	89	+	func run(args []string, h *harness) (err error) {
	90	+	opt, err := parseOptions(args)
	91	+	if err != nil {
	92	+	return err
	93	+	}
	94	+	opt, err = resolvePathOptions(opt)
	95	+	if err != nil {
	96	+	return err
	97	+	}
	98	+	m, err := readManifest(opt.manifest)
	99	+	if err != nil {
	100	+	return err
	101	+	}
	102	+	if err := validateManifest(m); err != nil {
	103	+	return err
	104	+	}
	105	+	if err = h.mkdir(opt.out); err != nil {
	106	+	return err
	107	+	}
	108	+	raw, err := os.OpenFile(filepath.Join(opt.out, "raw-harness.jsonl"), os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
	109	+	if err != nil {
	110	+	return err
	111	+	}
	112	+	defer func() {
	113	+	if closeErr := raw.Close(); closeErr != nil {
	114	+	err = errors.Join(err, closeErr)
	115	+	}
	116	+	}()
	117	+	results := make([]runResult, 0, len(m.Scenarios)*opt.repetitions)
	118	+	var localSpans []span
	119	+	scenarios := m.Scenarios
	120	+	if opt.scenario != "" {
	121	+	scenarios, err = selectScenario(m, opt.scenario)
	122	+	if err != nil {
	123	+	return err
	124	+	}
	125	+	}
	126	+	if h.selectScenarios != nil {
	127	+	scenarios = h.selectScenarios(m)
	128	+	}
	129	+	for _, s := range scenarios {
	130	+	if s.InapplicableReason != "" {
	131	+	continue
	132	+	}
	133	+	for r := 1; r <= opt.repetitions; r++ {
	134	+	rr, err := h.runOne(opt, m, s, r, raw)
	135	+	if err != nil {
	136	+	return fmt.Errorf("scenario %s run %d: %w", s.ID, r, err)
	137	+	}
	138	+	results = append(results, rr)
	139	+	localSpans = append(localSpans, rr.Spans...)
	140	+	}
	141	+	}
	142	+	all, cadence, maxGap, runP50s := aggregateEventResults(results)
	143	+	if len(all) == 0 {
	144	+	return errors.New("no measured harness input/flush pairs")
	145	+	}
	146	+	if err := writeJSON(filepath.Join(opt.out, "runs.json"), results); err != nil {
	147	+	return err
	148	+	}
	149	+	return writeJSON(filepath.Join(opt.out, "summary.json"), summary{Schema: 1, GitSHA: h.gitSHA(), Warmup: opt.warmup.String(), Duration: opt.duration.String(), Repetitions: opt.repetitions, EndToEnd: percentiles(all), Cadence: percentiles(cadence), MaxGap: maxGap, RunP50Dispersion: percentiles(runP50s), Spans: summarizeSpans(localSpans), Runs: len(results)})
	150	+	}
	151	+
	152	+	// aggregateEventResults deliberately preserves raw event latencies for
	153	+	// end-to-end percentiles. Per-run p50s are a separate dispersion series.
	154	+	func aggregateEventResults(results []runResult) (all, cadence []int64, maxGap int64, runP50s []int64) {
	155	+	for _, result := range results {
	156	+	all = append(all, result.EndToEnd...)
	157	+	cadence = append(cadence, result.CadenceSamples...)
	158	+	runP50s = append(runP50s, result.Event.P50)
	159	+	if result.MaxGap > maxGap {
	160	+	maxGap = result.MaxGap
	161	+	}
	162	+	}
	163	+	return all, cadence, maxGap, runP50s
	164	+	}
	165	+
	166	+	func (h *harness) runOne(o options, mat manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {
	167	+	dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))
	168	+	createRunDir := h.createRunDir
	169	+	if createRunDir == nil {
	170	+	createRunDir = createExclusiveRunDir
	171	+	}
	172	+	if err = createRunDir(dir); err != nil {
	173	+	return res, fmt.Errorf("create isolated run directory: %w", err)
	174	+	}
	175	+	success := false
	176	+	defer func() {
	177	+	if !success {
	178	+	if cleanupErr := h.removeAll(dir); cleanupErr != nil {
	179	+	err = errors.Join(err, fmt.Errorf("remove failed run directory: %w", cleanupErr))
	180	+	}
	181	+	}
	182	+	}()
	183	+	maps := make([]processMapping, 0, len(s.Roles))
	184	+	for i, role := range s.Roles {
	185	+	pid := fmt.Sprintf("%s-r%03d-%s-%02d", safeName(s.ID), run, safeName(role), i+1)
	186	+	path := filepath.Join(dir, pid+".jsonl")
	187	+	f, e := h.create(path)
	188	+	if e != nil {
	189	+	return res, fmt.Errorf("allocate exclusive trace %s: %w", role, e)
	190	+	}
	191	+	if e = f.Close(); e != nil {
	192	+	return res, e
	193	+	}
	194	+	maps = append(maps, processMapping{ProcessID: pid, ClockDomain: pid, TracePath: path, Role: role, Scenario: s.ID, Run: run, Identity: "harness-assigned:" + pid})
	195	+	}
	196	+	// Persist the complete mapping before any process can be launched. This is
	197	+	// both crash evidence and the only legal source for later ID correlation.
	198	+	if err = writeJSON(filepath.Join(dir, "manifest.json"), runManifest{Scenario: s.ID, Run: run, Processes: maps}); err != nil {
	199	+	return res, err
	200	+	}
	201	+	if h.launcher == nil {
	202	+	h.launcher = &cliLauncher{bin: o.vevBin}
	203	+	}
	204	+	processes := make([]launchedRole, 0, len(maps))
	205	+	closed := false
	206	+	closeRoles := func() error {
	207	+	if closed {
	208	+	return nil
	209	+	}
	210	+	closed = true
	211	+	// Clients own SSH-stdio descendants; close them before the deferred UDP
	212	+	// peer and daemon so no role survives a failed or completed run.
	213	+	return closeLaunchedRoles(processes)
	214	+	}
	215	+	defer func() {
	216	+	if cleanupErr := closeRoles(); cleanupErr != nil {
	217	+	if err == nil {
	218	+	err = cleanupErr
	219	+	} else {
	220	+	// Keep the primary workload failure first while retaining every
	221	+	// cleanup failure in reverse launch order.
	222	+	err = errors.Join(err, cleanupErr)
	223	+	}
	224	+	}
	225	+	}()
	226	+	selectedTransport, e := manifestTransport(mat, s.Transport)
	227	+	if e != nil {
	228	+	return res, e
	229	+	}
	230	+	for _, pm := range launchOrder(maps) {
	231	+	p, e := h.launcher.Launch(pm, routeRoleArgs(s, pm, selectedTransport))
	232	+	if e != nil {
	233	+	return res, e
	234	+	}
	235	+	processes = append(processes, launchedRole{mapping: pm, process: p})
	236	+	if pm.Role == "daemon" {
	237	+	if ready, ok := p.(interface{ WaitReady() error }); ok {
	238	+	if err = ready.WaitReady(); err != nil {
	239	+	return res, err
	240	+	}
	241	+	}
	242	+	}
	243	+	}
	244	+	marks := []harnessMark{}
	245	+	recordMark := func(mark harnessMark) error {
	246	+	marks = append(marks, mark)
	247	+	if err := json.NewEncoder(raw).Encode(mark); err != nil {
	248	+	return fmt.Errorf("write raw harness mark: %w", err)
	249	+	}
	250	+	return nil
	251	+	}
	252	+	seq := uint64(0)
	253	+	for _, p := range processes {
	254	+	if p.mapping.Role != "client" {
	255	+	continue
	256	+	}
	257	+	seq++
	258	+	sequence := seq
	259	+	if err = recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "input_injected", Tick: h.clock.Now(), Valid: true}); err != nil {
	260	+	return res, err
	261	+	}
	262	+	warmupErr := p.process.Warmup(workloadInput(s, run, "warmup"))
	263	+	if markErr := recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "terminal_flushed", Tick: h.clock.Now(), Valid: warmupErr == nil}); markErr != nil {
	264	+	return res, errors.Join(warmupErr, markErr)
	265	+	}
	266	+	if warmupErr != nil {
	267	+	return res, warmupErr
	268	+	}
	269	+	}
	270	+	// The harness clock, rather than a process clock, owns the explicit warmup
	271	+	// and measured intervals. Raw evidence retains every boundary, while only
	272	+	// complete valid pairs within [Start, End) contribute measurement samples.
	273	+	h.clock.Sleep(o.warmup)
	274	+	interval := measuredInterval{Start: h.clock.Now()}
	275	+	interval.End = interval.Start + o.duration.Nanoseconds()
	276	+	nextInjection := interval.Start
	277	+	for {
	278	+	now := h.clock.Now()
	279	+	if now < nextInjection {
	280	+	h.clock.Sleep(time.Duration(nextInjection - now))
	281	+	now = h.clock.Now()
	282	+	}
	283	+	if now >= interval.End {
	284	+	break
	285	+	}
	286	+	for _, p := range processes {
	287	+	if p.mapping.Role != "client" {
	288	+	continue
	289	+	}
	290	+	seq++
	291	+	sequence := seq
	292	+	injectedRecorded := false
	293	+	flushedRecorded := false
	294	+	injected := func() error {
	295	+	if markErr := recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "input_injected", Tick: h.clock.Now(), Valid: true}); markErr != nil {
	296	+	return markErr
	297	+	}
	298	+	injectedRecorded = true
	299	+	return nil
	300	+	}
	301	+	flushed := func() error {
	302	+	if markErr := recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "terminal_flushed", Tick: h.clock.Now(), Valid: true}); markErr != nil {
	303	+	return markErr
	304	+	}
	305	+	flushedRecorded = true
	306	+	return nil
	307	+	}
	308	+	if measureErr := p.process.Measure(workloadInput(s, run, fmt.Sprintf("measured-%d", sequence)), injected, flushed); measureErr != nil {
	309	+	if injectedRecorded && !flushedRecorded {
	310	+	diagnosticErr := recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "terminal_flushed", Tick: h.clock.Now(), Valid: false})
	311	+	return res, errors.Join(measureErr, diagnosticErr)
	312	+	}
	313	+	return res, measureErr
	314	+	}
	315	+	}
	316	+	// Cap injection rate rather than bursting events after a slow terminal
	317	+	// flush. Any resulting spacing is retained as raw cadence evidence.
	318	+	nextInjection = h.clock.Now() + measuredEventCadence.Nanoseconds()
	319	+	}
	320	+	if now := h.clock.Now(); now < interval.End {
	321	+	h.clock.Sleep(time.Duration(interval.End - now))
	322	+	}
	323	+	events, e := measuredEventSamples(marks, interval)
	324	+	if e != nil {
	325	+	return res, e
	326	+	}
	327	+	if err := requireMinimumEventSamples(events); err != nil {
	328	+	return res, err
	329	+	}
	330	+	samples := eventLatencies(events)
	331	+	cadenceSamples, maxGap := eventCadenceSamples(events)
	332	+	// Process teardown can unblock a receive and cause its failed end mark to
	333	+	// be serialized. Merge only the complete, post-cleanup per-process traces.
	334	+	if cleanupErr := closeRoles(); cleanupErr != nil {
	335	+	return res, cleanupErr
	336	+	}
	337	+	spans, e := mergeProcessTraces(maps)
	338	+	if e != nil {
	339	+	return res, e
	340	+	}
	341	+	success = true
	342	+	return runResult{Spans: spans, Scenario: s.ID, Run: run, Samples: len(samples), EndToEnd: samples, Event: percentiles(samples), Cadence: percentiles(cadenceSamples), CadenceSamples: cadenceSamples, MaxGap: maxGap, Processes: maps}, nil
	343	+	}
	344	+
	345	+	func roleArgs(s scenario, m processMapping) roleCommand {
	346	+	return routeRoleArgs(s, m, scenarioTransport(s))
	347	+	}
	348	+
	349	+	func routeRoleArgs(s scenario, m processMapping, selected transport) roleCommand {
	350	+	session := "perf-" + safeName(s.ID) + fmt.Sprintf("-%03d", m.Run)
	351	+	remote := []string{"attach", "harness@127.0.0.1:" + session}
	352	+	switch m.Role {
	353	+	case "daemon":
	354	+	return roleCommand{Args: []string{"--daemon"}}
	355	+	case "client":
	356	+	switch s.Transport {
	357	+	case "local":
	358	+	return roleCommand{Args: []string{"new", session}}
	359	+	case "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct":
	360	+	return roleCommand{Args: remote}
	361	+	}
	362	+	case "ssh_stdio_peer":
	363	+	return roleCommand{Args: []string{"_stdio", session}, Transport: transport{ID: s.Transport, Kind: "ssh_stdio"}}
	364	+	case "udp_peer":
	365	+	return roleCommand{Args: []string{"_udp-proxy", session}, Transport: selected}
	366	+	}
	367	+	return roleCommand{}
	368	+	}
	369	+
	370	+	func manifestTransport(m manifest, id string) (transport, error) {
	371	+	for _, t := range m.Transports {
	372	+	if t.ID == id {
	373	+	return t, nil
	374	+	}
	375	+	}
	376	+	// Focused unit tests may call runOne with a minimal manifest; production
	377	+	// already validates every scenario reference before this point.
	378	+	if m.Schema == 0 {
	379	+	return scenarioTransport(scenario{Transport: id}), nil
	380	+	}
	381	+	return transport{}, fmt.Errorf("scenario references missing transport %q", id)
	382	+	}
	383	+
	384	+	func scenarioTransport(s scenario) transport {
	385	+	switch s.Transport {
	386	+	case "udp_baseline":
	387	+	return transport{ID: s.Transport, Kind: "udp"}
	388	+	case "udp_25ms":
	389	+	return transport{ID: s.Transport, Kind: "udp", RTTMS: 25}
	390	+	case "udp_100ms":
	391	+	return transport{ID: s.Transport, Kind: "udp", RTTMS: 100}
	392	+	case "udp_loss_0pct":
	393	+	return transport{ID: s.Transport, Kind: "udp", LossPercent: 0}
	394	+	case "udp_loss_1pct":
	395	+	return transport{ID: s.Transport, Kind: "udp", LossPercent: 1}
	396	+	default:
	397	+	return transport{ID: s.Transport}
	398	+	}
	399	+	}
	400	+
	401	+	func launchOrder(mappings []processMapping) []processMapping {
	402	+	out := append([]processMapping(nil), mappings...)
	403	+	priority := func(role string) int {
	404	+	switch role {
	405	+	case "daemon":
	406	+	return 0
	407	+	case "ssh_stdio_peer", "udp_peer":
	408	+	return 1
	409	+	case "client":
	410	+	return 2
	411	+	default:
	412	+	return 3
	413	+	}
	414	+	}
	415	+	sort.SliceStable(out, func(i, j int) bool { return priority(out[i].Role) < priority(out[j].Role) })
	416	+	return out
	417	+	}
	418	+
	419	+	// workloadInput is shell input delivered through the client PTY. It avoids any
	420	+	// private daemon API and makes every topology/workload/transport manifest row
	421	+	// execute a real terminal workload. The marker is the output boundary awaited
	422	+	// by cliProcess; it is not a synthetic timestamp.
	423	+	func workloadInput(s scenario, run int, phase string) []byte {
	424	+	marker := fmt.Sprintf("__VEV_HARNESS_%s_r%d_%s__", safeName(s.ID), run, safeName(phase))
	425	+	body := "printf 'vev perf %s\\n'"
	426	+	switch s.Workload {
	427	+	case "active_output", "all_output", "inactive_output", "interactive_flood":
	428	+	body = "i=0; while [ $i -lt 128 ]; do printf 'vev perf output %s\\n'; i=$((i+1)); done"
	429	+	case "resize_sweep":
	430	+	body = "printf 'vev perf resize-sweep %s\\n'"
	431	+	case "copy_search":
	432	+	body = "printf 'vev perf copy-search %s\\n'"
	433	+	case "snapshot_output_resize":
	434	+	body = "printf 'vev perf snapshot-output-resize %s\\n'"
	435	+	case "attach_restore_tab_switch":
	436	+	body = "printf 'vev perf attach-restore-tab-switch %s\\n'"
	437	+	}
	438	+	return []byte(fmt.Sprintf(body+"; printf '%s\\n'\n", s.ID, marker))
	439	+	}

A cmd/vev-perf-harness/runner_test.go +456 -0

@@ -0,0 +1,456 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"os"
	9	+	"os/exec"
	10	+	"path/filepath"
	11	+	"strings"
	12	+	"testing"
	13	+	"time"
	14	+	)
	15	+
	16	+	func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
	17	+	if testing.Short() {
	18	+	t.Skip("public CLI repetition lifecycle")
	19	+	}
	20	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	21	+	if err != nil {
	22	+	t.Fatal(err)
	23	+	}
	24	+	bin := filepath.Join(t.TempDir(), "vev")
	25	+	build := exec.Command("go", "build", "-o", bin, "./")
	26	+	build.Dir = root
	27	+	if output, err := build.CombinedOutput(); err != nil {
	28	+	t.Fatalf("build public CLI: %v\n%s", err, output)
	29	+	}
	30	+	m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
	31	+	if err != nil {
	32	+	t.Fatal(err)
	33	+	}
	34	+	var local scenario
	35	+	for _, candidate := range m.Scenarios {
	36	+	if candidate.ID == "1x4-idle-local" {
	37	+	local = candidate
	38	+	break
	39	+	}
	40	+	}
	41	+	if local.ID == "" {
	42	+	t.Fatal("canonical local scenario missing")
	43	+	}
	44	+	out, err := os.MkdirTemp("", "vev-ob-")
	45	+	if err != nil {
	46	+	t.Fatal(err)
	47	+	}
	48	+	removeTestTree(t, out)
	49	+	raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	50	+	if err != nil {
	51	+	t.Fatal(err)
	52	+	}
	53	+	closeTestFile(t, raw)
	54	+	h := defaultHarness()
	55	+	h.clock = &fakeClock{}
	56	+	h.launcher = &cliLauncher{bin: bin}
	57	+	for run := 1; run <= 3; run++ {
	58	+	result, err := h.runOne(options{vevBin: bin, out: out, duration: minimumDuration}, m, local, run, raw)
	59	+	if err != nil {
	60	+	t.Fatalf("run %d: %v", run, err)
	61	+	}
	62	+	if len(result.Processes) != 2 {
	63	+	t.Fatalf("run %d process mappings = %+v", run, result.Processes)
	64	+	}
	65	+	runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
	66	+	manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
	67	+	if err != nil {
	68	+	t.Fatalf("run %d manifest: %v", run, err)
	69	+	}
	70	+	var persisted runManifest
	71	+	if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
	72	+	t.Fatalf("run %d manifest JSON: %v", run, err)
	73	+	}
	74	+	if persisted.Run != run \|\| len(persisted.Processes) != len(result.Processes) {
	75	+	t.Fatalf("run %d manifest = %+v", run, persisted)
	76	+	}
	77	+	for _, process := range result.Processes {
	78	+	if process.Role != "daemon" && process.Role != "client" {
	79	+	t.Fatalf("run %d unexpected role %q", run, process.Role)
	80	+	}
	81	+	if filepath.Dir(process.TracePath) != runDir {
	82	+	t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
	83	+	}
	84	+	trace, err := os.ReadFile(process.TracePath)
	85	+	if err != nil {
	86	+	t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
	87	+	}
	88	+	if len(bytes.TrimSpace(trace)) == 0 {
	89	+	t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
	90	+	}
	91	+	for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
	92	+	var record traceRecord
	93	+	if err := json.Unmarshal(line, &record); err != nil {
	94	+	t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
	95	+	}
	96	+	if record.ProcessID != process.ProcessID \|\| record.Scenario != local.ID \|\| record.Run != uint64(run) {
	97	+	t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
	98	+	}
	99	+	}
	100	+	}
	101	+	if _, err := os.Stat(filepath.Join(runDir, "runtime", "vev", "daemon.sock")); !errors.Is(err, os.ErrNotExist) {
	102	+	t.Fatalf("run %d left daemon socket for the next repetition: %v", run, err)
	103	+	}
	104	+	}
	105	+	}
	106	+
	107	+	func TestHarnessResolvesCanonicalRelativePathsBeforeRoleCWDChanges(t *testing.T) {
	108	+	if testing.Short() {
	109	+	t.Skip("public CLI relative-path lifecycle")
	110	+	}
	111	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	112	+	if err != nil {
	113	+	t.Fatal(err)
	114	+	}
	115	+	// Keep the real Unix-socket path below its platform limit while testing the
	116	+	// exact documented relative output hierarchy.
	117	+	workspace, err := os.MkdirTemp("", "vev-rel-")
	118	+	if err != nil {
	119	+	t.Fatal(err)
	120	+	}
	121	+	removeTestTree(t, workspace)
	122	+	bin := filepath.Join(workspace, "vev")
	123	+	build := exec.Command("go", "build", "-o", bin, "./")
	124	+	build.Dir = root
	125	+	if output, err := build.CombinedOutput(); err != nil {
	126	+	t.Fatalf("build public CLI: %v\n%s", err, output)
	127	+	}
	128	+	manifestPath := filepath.Join(workspace, "testdata", "perf", "manifest.json")
	129	+	if err := os.MkdirAll(filepath.Dir(manifestPath), 0o700); err != nil {
	130	+	t.Fatal(err)
	131	+	}
	132	+	contents, err := os.ReadFile(filepath.Join(root, "testdata", "perf", "manifest.json"))
	133	+	if err != nil {
	134	+	t.Fatal(err)
	135	+	}
	136	+	if err := os.WriteFile(manifestPath, contents, 0o600); err != nil {
	137	+	t.Fatal(err)
	138	+	}
	139	+	originalWD, err := os.Getwd()
	140	+	if err != nil {
	141	+	t.Fatal(err)
	142	+	}
	143	+	if err := os.Chdir(workspace); err != nil {
	144	+	t.Fatal(err)
	145	+	}
	146	+	t.Cleanup(func() {
	147	+	if err := os.Chdir(originalWD); err != nil {
	148	+	t.Errorf("restore cwd: %v", err)
	149	+	}
	150	+	})
	151	+
	152	+	// These are the documented relative locations. cliLauncher changes every
	153	+	// role to its run directory, so a failure to resolve them before launch
	154	+	// would make VEV_PERF_TRACE point at a nonexistent nested directory.
	155	+	h := defaultHarness()
	156	+	h.clock = &fakeClock{}
	157	+	if err := run([]string{
	158	+	"--vev-bin", "./vev",
	159	+	"--manifest", "testdata/perf/manifest.json",
	160	+	"--out", "testdata/perf/results",
	161	+	"--scenario", "1x4-idle-local",
	162	+	"--warmup", "0s",
	163	+	"--duration", "30s",
	164	+	"--repetitions", "10",
	165	+	}, h); err != nil {
	166	+	t.Fatal(err)
	167	+	}
	168	+
	169	+	out := filepath.Join(workspace, "testdata", "perf", "results")
	170	+	for run := 1; run <= 10; run++ {
	171	+	runDir := filepath.Join(out, fmt.Sprintf("1x4-idle-local-run-%03d", run))
	172	+	manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
	173	+	if err != nil {
	174	+	t.Fatalf("run %d manifest: %v", run, err)
	175	+	}
	176	+	var persisted runManifest
	177	+	if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
	178	+	t.Fatalf("run %d manifest JSON: %v", run, err)
	179	+	}
	180	+	if persisted.Run != run \|\| len(persisted.Processes) != 2 {
	181	+	t.Fatalf("run %d manifest=%+v", run, persisted)
	182	+	}
	183	+	for _, process := range persisted.Processes {
	184	+	if !filepath.IsAbs(process.TracePath) \|\| filepath.Dir(process.TracePath) != runDir {
	185	+	t.Fatalf("run %d %s trace path=%q, want absolute path in %q", run, process.Role, process.TracePath, runDir)
	186	+	}
	187	+	trace, err := os.ReadFile(process.TracePath)
	188	+	if err != nil \|\| len(bytes.TrimSpace(trace)) == 0 {
	189	+	t.Fatalf("run %d %s trace=%q err=%v", run, process.Role, process.TracePath, err)
	190	+	}
	191	+	}
	192	+	}
	193	+	}
	194	+
	195	+	func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
	196	+	out := t.TempDir()
	197	+	raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	198	+	if err != nil {
	199	+	t.Fatal(err)
	200	+	}
	201	+	closeTestFile(t, raw)
	202	+	s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
	203	+	stale := filepath.Join(out, "isolated-run-001")
	204	+	if err := os.Mkdir(stale, 0o700); err != nil {
	205	+	t.Fatal(err)
	206	+	}
	207	+	if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
	208	+	t.Fatal(err)
	209	+	}
	210	+	launcher := &fakeLauncher{}
	211	+	h := defaultHarness()
	212	+	h.clock, h.launcher = &fakeClock{}, launcher
	213	+	_, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
	214	+	if err == nil \|\| !strings.Contains(err.Error(), "create isolated run directory") {
	215	+	t.Fatalf("existing run directory error = %v", err)
	216	+	}
	217	+	if len(launcher.mappings) != 0 {
	218	+	t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
	219	+	}
	220	+	if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil \|\| string(contents) != "stale" {
	221	+	t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
	222	+	}
	223	+	}
	224	+
	225	+	func TestHarnessDoesNotInheritNestedSession(t *testing.T) {
	226	+	env := withoutEnv([]string{"VEV=session=old", "PATH=/bin", "VEV_PERF_TRACE=old"}, "VEV")
	227	+	if !equalStrings(env, []string{"PATH=/bin", "VEV_PERF_TRACE=old"}) {
	228	+	t.Fatalf("environment=%q", env)
	229	+	}
	230	+	}
	231	+
	232	+	func TestHarnessRoutesEveryRemoteFixtureThroughItsDeclaredPeer(t *testing.T) {
	233	+	cases := []struct {
	234	+	name string
	235	+	transport transport
	236	+	peer string
	237	+	wantRTT int
	238	+	wantLoss int
	239	+	}{
	240	+	{"ssh", transport{ID: "ssh_stdio", Kind: "ssh_stdio"}, "ssh_stdio_peer", 0, 0},
	241	+	{"udp baseline", transport{ID: "udp_baseline", Kind: "udp"}, "udp_peer", 0, 0},
	242	+	{"udp 25ms", transport{ID: "udp_25ms", Kind: "udp", RTTMS: 25}, "udp_peer", 25, 0},
	243	+	{"udp 100ms", transport{ID: "udp_100ms", Kind: "udp", RTTMS: 100}, "udp_peer", 100, 0},
	244	+	{"udp loss zero", transport{ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0}, "udp_peer", 0, 0},
	245	+	{"udp loss", transport{ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1}, "udp_peer", 0, 1},
	246	+	}
	247	+	for _, tc := range cases {
	248	+	t.Run(tc.name, func(t *testing.T) {
	249	+	s := scenario{ID: "route", Transport: tc.transport.ID}
	250	+	client := routeRoleArgs(s, processMapping{Role: "client", Run: 2}, tc.transport)
	251	+	peer := routeRoleArgs(s, processMapping{Role: tc.peer, Run: 2}, tc.transport)
	252	+	if !equalStrings(client.Args, []string{"attach", "harness@127.0.0.1:perf-route-002"}) {
	253	+	t.Fatalf("client did not use public remote attach: %q", client.Args)
	254	+	}
	255	+	if tc.peer == "ssh_stdio_peer" && !equalStrings(peer.Args, []string{"_stdio", "perf-route-002"}) {
	256	+	t.Fatalf("ssh peer command=%q", peer.Args)
	257	+	}
	258	+	if tc.peer == "udp_peer" && !equalStrings(peer.Args, []string{"_udp-proxy", "perf-route-002"}) {
	259	+	t.Fatalf("udp peer command=%q", peer.Args)
	260	+	}
	261	+	if peer.Transport.RTTMS != tc.wantRTT \|\| peer.Transport.LossPercent != tc.wantLoss {
	262	+	t.Fatalf("peer lost manifest network settings: %+v", peer.Transport)
	263	+	}
	264	+	})
	265	+	}
	266	+	}
	267	+
	268	+	func TestHarnessFakeRunnerRoutesClientToPeerAndCleansEveryRole(t *testing.T) {
	269	+	for _, tc := range []struct {
	270	+	name, transport, peer, peerCommand string
	271	+	}{
	272	+	{"ssh", "ssh_stdio", "ssh_stdio_peer", "_stdio"},
	273	+	{"udp baseline", "udp_baseline", "udp_peer", "_udp-proxy"},
	274	+	{"udp 25ms", "udp_25ms", "udp_peer", "_udp-proxy"},
	275	+	{"udp 100ms", "udp_100ms", "udp_peer", "_udp-proxy"},
	276	+	{"udp loss zero", "udp_loss_0pct", "udp_peer", "_udp-proxy"},
	277	+	{"udp loss", "udp_loss_1pct", "udp_peer", "_udp-proxy"},
	278	+	} {
	279	+	t.Run(tc.name, func(t *testing.T) {
	280	+	dir := t.TempDir()
	281	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	282	+	if err != nil {
	283	+	t.Fatal(err)
	284	+	}
	285	+	closeTestFile(t, raw)
	286	+	l := &fakeLauncher{}
	287	+	h := defaultHarness()
	288	+	h.clock, h.launcher = &fakeClock{}, l
	289	+	s := scenario{ID: "routed", Transport: tc.transport, Roles: []string{"daemon", "client", tc.peer}}
	290	+	if _, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, s, 1, raw); err != nil {
	291	+	t.Fatal(err)
	292	+	}
	293	+	if len(l.mappings) != 3 \|\| l.mappings[1].Role != tc.peer \|\| l.mappings[2].Role != "client" {
	294	+	t.Fatalf("dependency launch order=%+v", l.mappings)
	295	+	}
	296	+	if l.commands[1].Args[0] != tc.peerCommand \|\| l.commands[2].Args[0] != "attach" {
	297	+	t.Fatalf("commands do not connect client through declared peer: %+v", l.commands)
	298	+	}
	299	+	for i, p := range l.process {
	300	+	if !p.closed {
	301	+	t.Fatalf("role %s was not cleaned up", l.mappings[i].Role)
	302	+	}
	303	+	}
	304	+	})
	305	+	}
	306	+	}
	307	+
	308	+	func TestHarnessAggregatesRawEventsSeparatelyFromRunDispersion(t *testing.T) {
	309	+	first := append([]int64(nil), make([]int64, minimumInIntervalEventSamples+5)...)
	310	+	for i := range first {
	311	+	first[i] = 1
	312	+	}
	313	+	second := []int64{100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1_000, 1_000}
	314	+	all, _, _, runP50s := aggregateEventResults([]runResult{
	315	+	{EndToEnd: first, Event: distribution{P50: 1}},
	316	+	{EndToEnd: second, Event: distribution{P50: 100}},
	317	+	})
	318	+	if got, dispersion := percentiles(all), percentiles(runP50s); got.Count != 30 \|\| got.P99 != 1_000 \|\| dispersion.Count != 2 \|\| dispersion.P99 != 1 {
	319	+	t.Fatalf("event aggregation=%+v run dispersion=%+v", got, dispersion)
	320	+	}
	321	+	}
	322	+
	323	+	func TestHarnessRawRecordsContainEveryMarkWhileSamplesRemainFiltered(t *testing.T) {
	324	+	dir := t.TempDir()
	325	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	326	+	if err != nil {
	327	+	t.Fatal(err)
	328	+	}
	329	+	h := defaultHarness()
	330	+	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
	331	+	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "raw-complete", Roles: []string{"daemon", "client"}}, 1, raw)
	332	+	if err != nil {
	333	+	t.Fatal(err)
	334	+	}
	335	+	if err := raw.Close(); err != nil {
	336	+	t.Fatal(err)
	337	+	}
	338	+	marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
	339	+	if got, want := len(marks), 2*(result.Samples+1); got != want {
	340	+	t.Fatalf("raw marks=%d, want warmup plus every measured boundary=%d", got, want)
	341	+	}
	342	+	for i, mark := range marks[:2] {
	343	+	if mark.Sequence != 1 \|\| !mark.Valid \|\| []string{"input_injected", "terminal_flushed"}[i] != mark.Kind {
	344	+	t.Fatalf("warmup mark %d=%+v", i, mark)
	345	+	}
	346	+	}
	347	+	for sequence := uint64(2); sequence <= uint64(result.Samples+1); sequence++ {
	348	+	if marks[2(sequence-1)].Sequence != sequence \|\| marks[2(sequence-1)+1].Sequence != sequence {
	349	+	t.Fatalf("raw marks did not retain both boundaries for sequence %d: %+v", sequence, marks)
	350	+	}
	351	+	}
	352	+	}
	353	+
	354	+	func TestHarnessPersistsFailedFlushDiagnostics(t *testing.T) {
	355	+	dir := t.TempDir()
	356	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	357	+	if err != nil {
	358	+	t.Fatal(err)
	359	+	}
	360	+	failure := errors.New("flush failed")
	361	+	h := defaultHarness()
	362	+	h.clock, h.launcher = &fakeClock{}, failedFlushLauncher{err: failure}
	363	+	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "failed-flush", Roles: []string{"daemon", "client"}}, 1, raw)
	364	+	if !errors.Is(err, failure) {
	365	+	t.Fatalf("run error=%v, want failed flush", err)
	366	+	}
	367	+	if err := raw.Close(); err != nil {
	368	+	t.Fatal(err)
	369	+	}
	370	+	marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
	371	+	if got, want := len(marks), 4; got != want {
	372	+	t.Fatalf("raw marks=%d, want warmup and failed measured pair=%d: %+v", got, want, marks)
	373	+	}
	374	+	failed := marks[len(marks)-1]
	375	+	if failed.Sequence != 2 \|\| failed.Kind != "terminal_flushed" \|\| failed.Valid {
	376	+	t.Fatalf("failed flush diagnostic=%+v", failed)
	377	+	}
	378	+	}
	379	+
	380	+	func TestHarnessSchedulesMeasuredEventsAcrossFullInterval(t *testing.T) {
	381	+	dir := t.TempDir()
	382	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	383	+	if err != nil {
	384	+	t.Fatal(err)
	385	+	}
	386	+	closeTestFile(t, raw)
	387	+	h := defaultHarness()
	388	+	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
	389	+	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "schedule", Roles: []string{"daemon", "client"}}, 1, raw)
	390	+	if err != nil {
	391	+	t.Fatal(err)
	392	+	}
	393	+	if result.Samples < minimumInIntervalEventSamples \|\| result.Cadence.Count != result.Samples-1 \|\| result.MaxGap <= 0 {
	394	+	t.Fatalf("result did not include repeated measured events: %+v", result)
	395	+	}
	396	+	if err := raw.Close(); err != nil {
	397	+	t.Fatal(err)
	398	+	}
	399	+	records, err := readJSONL(filepath.Join(dir, "raw.jsonl"))
	400	+	if err != nil {
	401	+	t.Fatal(err)
	402	+	}
	403	+	if want := result.Samples*2 + 2; len(records) != want {
	404	+	t.Fatalf("raw records=%d, want %d warmup and measured boundaries", len(records), want)
	405	+	}
	406	+	}
	407	+
	408	+	func TestHarnessPreservesWorkloadAndCleanupErrors(t *testing.T) {
	409	+	dir := t.TempDir()
	410	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	411	+	if err != nil {
	412	+	t.Fatal(err)
	413	+	}
	414	+	closeTestFile(t, raw)
	415	+	primary := errors.New("primary workload failure")
	416	+	cleanup := errors.New("daemon cleanup failure")
	417	+	launcher := &fakeLauncher{
	418	+	measureErr: map[string]error{"client": primary},
	419	+	closeErr: map[string]error{"daemon": cleanup},
	420	+	}
	421	+	h := defaultHarness()
	422	+	h.clock, h.launcher = &fakeClock{}, launcher
	423	+	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "errors", Roles: []string{"daemon", "client"}}, 1, raw)
	424	+	if !errors.Is(err, primary) \|\| !errors.Is(err, cleanup) {
	425	+	t.Fatalf("workload and cleanup errors were not both preserved: %v", err)
	426	+	}
	427	+	if got, want := err.Error(), primary.Error()+"\nclose daemon role: "+cleanup.Error(); got != want {
	428	+	t.Fatalf("error ordering=%q want %q", got, want)
	429	+	}
	430	+	for i, p := range launcher.process {
	431	+	if !p.closed {
	432	+	t.Fatalf("role %s was not closed", launcher.mappings[i].Role)
	433	+	}
	434	+	}
	435	+	}
	436	+
	437	+	func TestHarnessCleansRunDirectoryOnProcessFailure(t *testing.T) {
	438	+	dir := t.TempDir()
	439	+	raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	440	+	if err != nil {
	441	+	t.Fatal(err)
	442	+	}
	443	+	closeTestFile(t, raw)
	444	+	launcher := &fakeLauncher{}
	445	+	h := defaultHarness()
	446	+	h.clock = &fakeClock{}
	447	+	// The client is the second role and fails after the daemon has started.
	448	+	h.launcher = &failingLauncher{fakeLauncher: launcher}
	449	+	o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
	450	+	if _, err := h.runOne(o, manifest{}, scenario{ID: "cleanup", Roles: []string{"daemon", "client"}}, 1, raw); err == nil {
	451	+	t.Fatal("failed process accepted")
	452	+	}
	453	+	if _, err := os.Stat(filepath.Join(dir, "cleanup-run-001")); !os.IsNotExist(err) {
	454	+	t.Fatalf("failed run directory remains: %v", err)
	455	+	}
	456	+	}

A cmd/vev-perf-harness/test_helpers_test.go +341 -0

@@ -0,0 +1,341 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"os"
	9	+	"path/filepath"
	10	+	"testing"
	11	+	"time"
	12	+	)
	13	+
	14	+	type fakeClock struct{ tick int64 }
	15	+
	16	+	func (c *fakeClock) Now() int64 { c.tick += 10; return c.tick }
	17	+
	18	+	func (c *fakeClock) Sleep(d time.Duration) {
	19	+	c.tick += d.Nanoseconds()
	20	+	}
	21	+
	22	+	type fakeUDPNetem struct {
	23	+	port int
	24	+	closed bool
	25	+	}
	26	+
	27	+	func (n *fakeUDPNetem) Port() int { return n.port }
	28	+
	29	+	func (n *fakeUDPNetem) Close() error { n.closed = true; return nil }
	30	+
	31	+	type fakeProcess struct {
	32	+	fail bool
	33	+	measureErr error
	34	+	closeErr error
	35	+	warmups [][]byte
	36	+	measures [][]byte
	37	+	closed bool
	38	+	}
	39	+
	40	+	func (p *fakeProcess) Warmup(input []byte) error {
	41	+	p.warmups = append(p.warmups, append([]byte(nil), input...))
	42	+	return nil
	43	+	}
	44	+
	45	+	func (p *fakeProcess) Measure(input []byte, injected, flush func() error) error {
	46	+	p.measures = append(p.measures, append([]byte(nil), input...))
	47	+	if p.measureErr != nil {
	48	+	return p.measureErr
	49	+	}
	50	+	if p.fail {
	51	+	return errors.New("measure failure")
	52	+	}
	53	+	if err := injected(); err != nil {
	54	+	return err
	55	+	}
	56	+	return flush()
	57	+	}
	58	+
	59	+	func (p *fakeProcess) Close() error { p.closed = true; return p.closeErr }
	60	+
	61	+	type fakeLauncher struct {
	62	+	mappings []processMapping
	63	+	args [][]string
	64	+	commands []roleCommand
	65	+	process []*fakeProcess
	66	+	measureErr map[string]error
	67	+	closeErr map[string]error
	68	+	manifestPresent bool
	69	+	}
	70	+
	71	+	func (l *fakeLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	72	+	l.mappings = append(l.mappings, m)
	73	+	l.args = append(l.args, append([]string(nil), command.Args...))
	74	+	l.commands = append(l.commands, command)
	75	+	_, e := os.Stat(filepath.Join(filepath.Dir(m.TracePath), "manifest.json"))
	76	+	l.manifestPresent = e == nil
	77	+	p := &fakeProcess{measureErr: l.measureErr[m.Role], closeErr: l.closeErr[m.Role]}
	78	+	l.process = append(l.process, p)
	79	+	return p, nil
	80	+	}
	81	+
	82	+	type lateFlushProcess struct{ clock *fakeClock }
	83	+
	84	+	func (*lateFlushProcess) Warmup([]byte) error { return nil }
	85	+
	86	+	func (p *lateFlushProcess) Measure(_ []byte, injected, flushed func() error) error {
	87	+	if err := injected(); err != nil {
	88	+	return err
	89	+	}
	90	+	p.clock.tick += minimumDuration.Nanoseconds()
	91	+	return flushed()
	92	+	}
	93	+
	94	+	func (*lateFlushProcess) Close() error { return nil }
	95	+
	96	+	type insufficientEventLauncher struct{ clock *fakeClock }
	97	+
	98	+	func (l insufficientEventLauncher) Launch(m processMapping, _ roleCommand) (launchedProcess, error) {
	99	+	if m.Role == "client" {
	100	+	return &lateFlushProcess{clock: l.clock}, nil
	101	+	}
	102	+	return &fakeProcess{}, nil
	103	+	}
	104	+
	105	+	type failedFlushProcess struct{ err error }
	106	+
	107	+	func (*failedFlushProcess) Warmup([]byte) error { return nil }
	108	+
	109	+	func (p *failedFlushProcess) Measure(_ []byte, injected, _ func() error) error {
	110	+	if err := injected(); err != nil {
	111	+	return err
	112	+	}
	113	+	return p.err
	114	+	}
	115	+
	116	+	func (*failedFlushProcess) Close() error { return nil }
	117	+
	118	+	type failedFlushLauncher struct{ err error }
	119	+
	120	+	func (l failedFlushLauncher) Launch(m processMapping, _ roleCommand) (launchedProcess, error) {
	121	+	if m.Role == "client" {
	122	+	return &failedFlushProcess{err: l.err}, nil
	123	+	}
	124	+	return &fakeProcess{}, nil
	125	+	}
	126	+
	127	+	func closeTestFile(t testing.T, f os.File) {
	128	+	t.Helper()
	129	+	t.Cleanup(func() {
	130	+	if err := f.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
	131	+	t.Error(err)
	132	+	}
	133	+	})
	134	+	}
	135	+
	136	+	func removeTestTree(t *testing.T, path string) {
	137	+	t.Helper()
	138	+	t.Cleanup(func() {
	139	+	if err := os.RemoveAll(path); err != nil {
	140	+	t.Error(err)
	141	+	}
	142	+	})
	143	+	}
	144	+
	145	+	func readHarnessMarks(t *testing.T, path string) []harnessMark {
	146	+	t.Helper()
	147	+	records, err := readJSONL(path)
	148	+	if err != nil {
	149	+	t.Fatal(err)
	150	+	}
	151	+	marks := make([]harnessMark, len(records))
	152	+	for i, record := range records {
	153	+	encoded, err := json.Marshal(record)
	154	+	if err != nil {
	155	+	t.Fatal(err)
	156	+	}
	157	+	if err := json.Unmarshal(encoded, &marks[i]); err != nil {
	158	+	t.Fatal(err)
	159	+	}
	160	+	}
	161	+	return marks
	162	+	}
	163	+
	164	+	type fakePTY struct{ writes [][]byte }
	165	+
	166	+	func (p *fakePTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
	167	+
	168	+	func (p *fakePTY) Write(b []byte) (int, error) {
	169	+	p.writes = append(p.writes, append([]byte(nil), b...))
	170	+	return len(b), nil
	171	+	}
	172	+
	173	+	func (*fakePTY) Close() error { return nil }
	174	+
	175	+	type stagedPTY struct {
	176	+	allow <-chan struct{}
	177	+	writes chan []byte
	178	+	}
	179	+
	180	+	func (*stagedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
	181	+
	182	+	func (p *stagedPTY) Write(b []byte) (int, error) {
	183	+	if p.allow != nil {
	184	+	<-p.allow
	185	+	}
	186	+	p.writes <- append([]byte(nil), b...)
	187	+	return len(b), nil
	188	+	}
	189	+
	190	+	func (*stagedPTY) Close() error { return nil }
	191	+
	192	+	type fakeOutput struct {
	193	+	bytes.Buffer
	194	+	syncs int
	195	+	}
	196	+
	197	+	func (o *fakeOutput) Sync() error { o.syncs++; return nil }
	198	+
	199	+	func (*fakeOutput) Close() error { return nil }
	200	+
	201	+	type orderedPTY struct {
	202	+	write func([]byte)
	203	+	close func()
	204	+	}
	205	+
	206	+	func (*orderedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
	207	+
	208	+	func (p *orderedPTY) Write(b []byte) (int, error) {
	209	+	if p.write != nil {
	210	+	p.write(b)
	211	+	}
	212	+	return len(b), nil
	213	+	}
	214	+
	215	+	func (p *orderedPTY) Close() error {
	216	+	p.close()
	217	+	return nil
	218	+	}
	219	+
	220	+	type orderedOutput struct{ close func() }
	221	+
	222	+	func (*orderedOutput) Write(b []byte) (int, error) { return len(b), nil }
	223	+
	224	+	func (*orderedOutput) Sync() error { return nil }
	225	+
	226	+	func (o *orderedOutput) Close() error {
	227	+	o.close()
	228	+	return nil
	229	+	}
	230	+
	231	+	type failingLauncher struct{ *fakeLauncher }
	232	+
	233	+	func (l *failingLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	234	+	p, err := l.fakeLauncher.Launch(m, command)
	235	+	if err != nil {
	236	+	return nil, err
	237	+	}
	238	+	if m.Role == "client" {
	239	+	fake, ok := p.(*fakeProcess)
	240	+	if !ok {
	241	+	return nil, fmt.Errorf("fake launcher returned %T, want *fakeProcess", p)
	242	+	}
	243	+	fake.fail = true
	244	+	}
	245	+	return p, nil
	246	+	}
	247	+
	248	+	func equalStrings(a, b []string) bool {
	249	+	if len(a) != len(b) {
	250	+	return false
	251	+	}
	252	+	for i := range a {
	253	+	if a[i] != b[i] {
	254	+	return false
	255	+	}
	256	+	}
	257	+	return true
	258	+	}
	259	+
	260	+	type traceLauncher struct{ fakeLauncher }
	261	+
	262	+	func (l *traceLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	263	+	p, err := l.fakeLauncher.Launch(m, command)
	264	+	if err != nil {
	265	+	return nil, err
	266	+	}
	267	+	f, err := os.OpenFile(m.TracePath, os.O_WRONLY\|os.O_APPEND, 0o600)
	268	+	if err != nil {
	269	+	return nil, err
	270	+	}
	271	+	for i, pair := range spanPairs {
	272	+	for n, kind := range []string{pair.start, pair.end} {
	273	+	b, err := json.Marshal(traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: uint64(i + 1), RequestID: 1, Epoch: 1, Kind: kind, Tick: int64(i20 + n5), Valid: true})
	274	+	if err != nil {
	275	+	return nil, errors.Join(err, f.Close())
	276	+	}
	277	+	if _, err := f.Write(append(b, '\n')); err != nil {
	278	+	return nil, errors.Join(err, f.Close())
	279	+	}
	280	+	}
	281	+	}
	282	+	if err := f.Close(); err != nil {
	283	+	return nil, err
	284	+	}
	285	+	return p, nil
	286	+	}
	287	+
	288	+	type closeTraceProcess struct {
	289	+	*fakeProcess
	290	+	traceEnd func() error
	291	+	}
	292	+
	293	+	func (p *closeTraceProcess) Close() error {
	294	+	p.closed = true
	295	+	return errors.Join(p.closeErr, p.traceEnd())
	296	+	}
	297	+
	298	+	type closeTraceLauncher struct{ fakeLauncher }
	299	+
	300	+	func (l *closeTraceLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	301	+	p, err := l.fakeLauncher.Launch(m, command)
	302	+	if err != nil {
	303	+	return nil, err
	304	+	}
	305	+	start := traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: 1, RequestID: 1, Epoch: 1, Kind: "adapter_receive_start", Tick: 10, Valid: true}
	306	+	if err := appendTraceRecord(m.TracePath, start); err != nil {
	307	+	return nil, err
	308	+	}
	309	+	fake, ok := p.(*fakeProcess)
	310	+	if !ok {
	311	+	return nil, fmt.Errorf("fake launcher returned %T, want *fakeProcess", p)
	312	+	}
	313	+	return &closeTraceProcess{fakeProcess: fake, traceEnd: func() error {
	314	+	return appendTraceRecord(m.TracePath, traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: 1, RequestID: 1, Epoch: 1, Kind: "adapter_receive_end", Tick: 20, Valid: true})
	315	+	}}, nil
	316	+	}
	317	+
	318	+	func appendTraceRecord(path string, record traceRecord) error {
	319	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_APPEND, 0o600)
	320	+	if err != nil {
	321	+	return err
	322	+	}
	323	+	b, err := json.Marshal(record)
	324	+	if err != nil {
	325	+	return errors.Join(err, f.Close())
	326	+	}
	327	+	if _, err := f.Write(append(b, '\n')); err != nil {
	328	+	return errors.Join(err, f.Close())
	329	+	}
	330	+	return f.Close()
	331	+	}
	332	+
	333	+	// boundedLocalConcurrentTrace is an excerpt from the second run of the
	334	+	// bounded local public-CLI smoke. Receive sequence 3 is in flight while the
	335	+	// same process serializes sequence 4; shared-process records therefore cannot
	336	+	// have a global sequence ordering requirement.
	337	+	const boundedLocalConcurrentTrace = `{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":3,"request_id":3,"epoch":3,"kind":"adapter_receive_start","tick":1783902621495284782,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	338	+	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":4,"request_id":4,"epoch":4,"kind":"adapter_send_start","tick":1783902621495364552,"bytes":96,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	339	+	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":4,"request_id":4,"epoch":4,"kind":"adapter_send_end","tick":1783902621495386452,"bytes":96,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	340	+	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":3,"request_id":3,"epoch":3,"kind":"adapter_receive_end","tick":1783902621495463332,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	341	+	`

A cmd/vev-perf-harness/udp_netem.go +271 -0

@@ -0,0 +1,271 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"container/heap"
	5	+	"errors"
	6	+	"fmt"
	7	+	"net"
	8	+	"os"
	9	+	"strconv"
	10	+	"strings"
	11	+	"sync"
	12	+	"sync/atomic"
	13	+	"time"
	14	+	)
	15	+
	16	+	// udpNetem is a harness-owned, user-space UDP relay. It sits between the
	17	+	// public dgram client and vev's public _udp-proxy readiness address, so fixture
	18	+	// impairment is real network carriage rather than an ignored vev option.
	19	+	type udpNetem interface {
	20	+	Port() int
	21	+	Close() error
	22	+	}
	23	+
	24	+	type udpNetemFactory func(udpNetemConfig) (udpNetem, error)
	25	+
	26	+	type udpNetemConfig struct {
	27	+	// RTT is split equally across both relay directions.
	28	+	RTT time.Duration
	29	+	LossPercent int
	30	+	TargetPath string
	31	+	}
	32	+
	33	+	// udpNetemQueueCapacity bounds both packets waiting to be scheduled and
	34	+	// packets waiting for their delivery deadline. When it is full, the relay
	35	+	// drops the arriving packet and records it in queueOverflowDrops.
	36	+	const udpNetemQueueCapacity = 256
	37	+
	38	+	type scheduledPacket struct {
	39	+	packet []byte
	40	+	to net.Addr
	41	+	due time.Time
	42	+	}
	43	+
	44	+	type scheduledPackets []scheduledPacket
	45	+
	46	+	func (p scheduledPackets) Len() int { return len(p) }
	47	+	func (p scheduledPackets) Less(i, j int) bool { return p[i].due.Before(p[j].due) }
	48	+	func (p scheduledPackets) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
	49	+	func (p *scheduledPackets) Push(x any) {
	50	+	packet, ok := x.(scheduledPacket)
	51	+	if !ok {
	52	+	panic(fmt.Sprintf("scheduled packet type %T", x))
	53	+	}
	54	+	p = append(p, packet)
	55	+	}
	56	+	func (p *scheduledPackets) Pop() any {
	57	+	old := *p
	58	+	n := len(old)
	59	+	item := old[n-1]
	60	+	*p = old[:n-1]
	61	+	return item
	62	+	}
	63	+
	64	+	type udpNetemRelay struct {
	65	+	config udpNetemConfig
	66	+	conn net.PacketConn
	67	+	done chan struct{}
	68	+	wg sync.WaitGroup
	69	+	once sync.Once
	70	+	closeErr error
	71	+
	72	+	// slots bounds queued and deadline-pending packets together. The reader and
	73	+	// the sole scheduler are the relay's only goroutines.
	74	+	slots chan struct{}
	75	+	queue chan scheduledPacket
	76	+	queueOverflowDrops atomic.Uint64
	77	+	deliveryErrors atomic.Uint64
	78	+
	79	+	mu sync.Mutex
	80	+	client, dst net.Addr
	81	+	packet uint64
	82	+	}
	83	+
	84	+	func newUDPNetem(config udpNetemConfig) (udpNetem, error) {
	85	+	if config.RTT < 0 \|\| config.LossPercent < 0 \|\| config.LossPercent > 100 \|\| config.TargetPath == "" {
	86	+	return nil, fmt.Errorf("invalid UDP netem fixture: rtt=%s loss=%d target=%q", config.RTT, config.LossPercent, config.TargetPath)
	87	+	}
	88	+	conn, err := net.ListenPacket("udp", "127.0.0.1:0")
	89	+	if err != nil {
	90	+	return nil, err
	91	+	}
	92	+	r := &udpNetemRelay{
	93	+	config: config,
	94	+	conn: conn,
	95	+	done: make(chan struct{}),
	96	+	slots: make(chan struct{}, udpNetemQueueCapacity),
	97	+	queue: make(chan scheduledPacket, udpNetemQueueCapacity),
	98	+	}
	99	+	r.wg.Add(2)
	100	+	go r.run()
	101	+	go r.schedule()
	102	+	return r, nil
	103	+	}
	104	+
	105	+	func (r *udpNetemRelay) Port() int {
	106	+	addr, ok := r.conn.LocalAddr().(*net.UDPAddr)
	107	+	if !ok {
	108	+	return 0
	109	+	}
	110	+	return addr.Port
	111	+	}
	112	+
	113	+	func (r *udpNetemRelay) Close() error {
	114	+	r.once.Do(func() {
	115	+	close(r.done)
	116	+	r.closeErr = r.conn.Close()
	117	+	r.wg.Wait()
	118	+	if drops := r.queueOverflowDrops.Load(); drops != 0 {
	119	+	r.closeErr = errors.Join(r.closeErr, fmt.Errorf("udp netem queue overflow drops: %d", drops))
	120	+	}
	121	+	if writes := r.deliveryErrors.Load(); writes != 0 {
	122	+	r.closeErr = errors.Join(r.closeErr, fmt.Errorf("udp netem delivery errors: %d", writes))
	123	+	}
	124	+	})
	125	+	return r.closeErr
	126	+	}
	127	+
	128	+	func (r *udpNetemRelay) run() {
	129	+	defer r.wg.Done()
	130	+	buf := make([]byte, 64*1024)
	131	+	for {
	132	+	n, from, err := r.conn.ReadFrom(buf)
	133	+	if err != nil {
	134	+	return
	135	+	}
	136	+	to, ok := r.destination(from)
	137	+	if !ok \|\| r.drop() {
	138	+	continue
	139	+	}
	140	+	// Copy because PacketConn reuses buf on its next read.
	141	+	r.enqueue(scheduledPacket{packet: append([]byte(nil), buf[:n]...), to: to, due: time.Now().Add(r.config.RTT / 2)})
	142	+	}
	143	+	}
	144	+
	145	+	// enqueue implements the explicit overflow policy: never block the reader;
	146	+	// drop a packet if the bounded scheduler capacity has been reached.
	147	+	func (r *udpNetemRelay) enqueue(packet scheduledPacket) bool {
	148	+	select {
	149	+	case r.slots <- struct{}{}:
	150	+	case <-r.done:
	151	+	return false
	152	+	default:
	153	+	r.queueOverflowDrops.Add(1)
	154	+	return false
	155	+	}
	156	+	select {
	157	+	case r.queue <- packet:
	158	+	return true
	159	+	case <-r.done:
	160	+	<-r.slots
	161	+	return false
	162	+	}
	163	+	}
	164	+
	165	+	func (r *udpNetemRelay) schedule() {
	166	+	defer r.wg.Done()
	167	+	pending := scheduledPackets{}
	168	+	heap.Init(&pending)
	169	+	timer := time.NewTimer(time.Hour)
	170	+	if !timer.Stop() {
	171	+	<-timer.C
	172	+	}
	173	+	defer timer.Stop()
	174	+	var timerC <-chan time.Time
	175	+	resetTimer := func() {
	176	+	if timerC != nil && !timer.Stop() {
	177	+	select {
	178	+	case <-timer.C:
	179	+	default:
	180	+	}
	181	+	}
	182	+	if pending.Len() == 0 {
	183	+	timerC = nil
	184	+	return
	185	+	}
	186	+	delay := time.Until(pending[0].due)
	187	+	if delay < 0 {
	188	+	delay = 0
	189	+	}
	190	+	timer.Reset(delay)
	191	+	timerC = timer.C
	192	+	}
	193	+	for {
	194	+	resetTimer()
	195	+	select {
	196	+	case <-r.done:
	197	+	return
	198	+	case packet := <-r.queue:
	199	+	heap.Push(&pending, packet)
	200	+	case <-timerC:
	201	+	now := time.Now()
	202	+	for pending.Len() > 0 && !pending[0].due.After(now) {
	203	+	packet, ok := heap.Pop(&pending).(scheduledPacket)
	204	+	if !ok {
	205	+	panic("scheduled packet heap contained unexpected type")
	206	+	}
	207	+	select {
	208	+	case <-r.done:
	209	+	return
	210	+	default:
	211	+	if _, err := r.conn.WriteTo(packet.packet, packet.to); err != nil {
	212	+	select {
	213	+	case <-r.done:
	214	+	default:
	215	+	r.deliveryErrors.Add(1)
	216	+	}
	217	+	}
	218	+	<-r.slots
	219	+	}
	220	+	}
	221	+	}
	222	+	}
	223	+	}
	224	+
	225	+	func (r *udpNetemRelay) destination(from net.Addr) (net.Addr, bool) {
	226	+	r.mu.Lock()
	227	+	defer r.mu.Unlock()
	228	+	if r.dst == nil {
	229	+	dst, err := udpNetemTarget(r.config.TargetPath)
	230	+	if err != nil {
	231	+	return nil, false
	232	+	}
	233	+	r.dst = dst
	234	+	}
	235	+	if sameUDPAddress(from, r.dst) {
	236	+	return r.client, r.client != nil
	237	+	}
	238	+	r.client = from
	239	+	return r.dst, true
	240	+	}
	241	+
	242	+	func (r *udpNetemRelay) drop() bool {
	243	+	r.mu.Lock()
	244	+	defer r.mu.Unlock()
	245	+	r.packet++
	246	+	// The deterministic cycle makes the configured percentage auditable: a 1%
	247	+	// fixture drops exactly one carriage packet in every 100 attempts.
	248	+	return r.config.LossPercent > 0 && int((r.packet-1)%100) < r.config.LossPercent
	249	+	}
	250	+
	251	+	func udpNetemTarget(path string) (net.Addr, error) {
	252	+	line, err := os.ReadFile(path)
	253	+	if err != nil {
	254	+	return nil, err
	255	+	}
	256	+	fields := strings.Fields(string(line))
	257	+	if len(fields) != 3 \|\| fields[0] != "VEV-UDP" {
	258	+	return nil, errors.New("invalid UDP proxy readiness")
	259	+	}
	260	+	port, err := strconv.Atoi(fields[1])
	261	+	if err != nil \|\| port < 1 \|\| port > 65535 \|\| fields[2] == "" {
	262	+	return nil, errors.New("invalid UDP proxy readiness")
	263	+	}
	264	+	return &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: port}, nil
	265	+	}
	266	+
	267	+	func sameUDPAddress(a, b net.Addr) bool {
	268	+	ua, aOK := a.(*net.UDPAddr)
	269	+	ub, bOK := b.(*net.UDPAddr)
	270	+	return aOK && bOK && ua.Port == ub.Port && ua.IP.Equal(ub.IP)
	271	+	}

A cmd/vev-perf-harness/udp_netem_test.go +147 -0

@@ -0,0 +1,147 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"net"
	5	+	"os"
	6	+	"path/filepath"
	7	+	"strconv"
	8	+	"testing"
	9	+	"time"
	10	+	)
	11	+
	12	+	func closeUDPFixture(t *testing.T, c interface{ Close() error }) {
	13	+	t.Helper()
	14	+	t.Cleanup(func() {
	15	+	if err := c.Close(); err != nil {
	16	+	t.Error(err)
	17	+	}
	18	+	})
	19	+	}
	20	+
	21	+	func TestUDPNetemDropsWhenBoundedSchedulerIsFull(t *testing.T) {
	22	+	r := &udpNetemRelay{
	23	+	done: make(chan struct{}),
	24	+	slots: make(chan struct{}, udpNetemQueueCapacity),
	25	+	queue: make(chan scheduledPacket, udpNetemQueueCapacity),
	26	+	}
	27	+	packet := scheduledPacket{packet: []byte("bounded"), to: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 1}}
	28	+	for range udpNetemQueueCapacity {
	29	+	if !r.enqueue(packet) {
	30	+	t.Fatal("enqueue() dropped packet before scheduler capacity was full")
	31	+	}
	32	+	}
	33	+	if r.enqueue(packet) {
	34	+	t.Fatal("enqueue() accepted packet beyond scheduler capacity")
	35	+	}
	36	+	if got := r.queueOverflowDrops.Load(); got != 1 {
	37	+	t.Fatalf("queue overflow drops=%d, want 1", got)
	38	+	}
	39	+	}
	40	+
	41	+	func TestUDPNetemCloseRejectsQueueOverflow(t *testing.T) {
	42	+	netem, err := newUDPNetem(udpNetemConfig{TargetPath: filepath.Join(t.TempDir(), "target")})
	43	+	if err != nil {
	44	+	t.Fatal(err)
	45	+	}
	46	+	relay, ok := netem.(*udpNetemRelay)
	47	+	if !ok {
	48	+	t.Fatalf("netem type=%T, want *udpNetemRelay", netem)
	49	+	}
	50	+	relay.queueOverflowDrops.Store(3)
	51	+	if err := netem.Close(); err == nil \|\| err.Error() != "udp netem queue overflow drops: 3" {
	52	+	t.Fatalf("Close error=%v, want overflow rejection", err)
	53	+	}
	54	+	}
	55	+
	56	+	func TestUDPNetemExecutesRTTAndLossFixtures(t *testing.T) {
	57	+	fixtures := []struct {
	58	+	name string
	59	+	rtt time.Duration
	60	+	loss int
	61	+	sent int
	62	+	want int
	63	+	}{
	64	+	{"25ms RTT", 25 * time.Millisecond, 0, 1, 1},
	65	+	{"100ms RTT", 100 * time.Millisecond, 0, 1, 1},
	66	+	{"zero loss", 0, 0, 10, 10},
	67	+	{"one percent loss", 0, 1, 100, 99},
	68	+	}
	69	+	for _, tc := range fixtures {
	70	+	t.Run(tc.name, func(t *testing.T) {
	71	+	target, err := net.ListenPacket("udp", "127.0.0.1:0")
	72	+	if err != nil {
	73	+	t.Fatal(err)
	74	+	}
	75	+	closeUDPFixture(t, target)
	76	+	targetAddr, ok := target.LocalAddr().(*net.UDPAddr)
	77	+	if !ok {
	78	+	t.Fatalf("target address type=%T, want *net.UDPAddr", target.LocalAddr())
	79	+	}
	80	+	path := filepath.Join(t.TempDir(), "target")
	81	+	if err := os.WriteFile(path, []byte("VEV-UDP "+strconv.Itoa(targetAddr.Port)+" key\\n"), 0o600); err != nil {
	82	+	t.Fatal(err)
	83	+	}
	84	+	netem, err := newUDPNetem(udpNetemConfig{RTT: tc.rtt, LossPercent: tc.loss, TargetPath: path})
	85	+	if err != nil {
	86	+	t.Fatal(err)
	87	+	}
	88	+	closeUDPFixture(t, netem)
	89	+	client, err := net.ListenPacket("udp", "127.0.0.1:0")
	90	+	if err != nil {
	91	+	t.Fatal(err)
	92	+	}
	93	+	closeUDPFixture(t, client)
	94	+	started := time.Now()
	95	+	for i := 0; i < tc.sent; i++ {
	96	+	if _, err := client.WriteTo([]byte{byte(i)}, &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: netem.Port()}); err != nil {
	97	+	t.Fatal(err)
	98	+	}
	99	+	}
	100	+	if err := target.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
	101	+	t.Fatal(err)
	102	+	}
	103	+	got := 0
	104	+	buf := make([]byte, 32)
	105	+	var relayAddr net.Addr
	106	+	for got < tc.want {
	107	+	if _, relayAddr, err = target.ReadFrom(buf); err != nil {
	108	+	t.Fatalf("received %d packets, want %d: %v", got, tc.want, err)
	109	+	}
	110	+	got++
	111	+	}
	112	+	if tc.rtt > 0 {
	113	+	if elapsed := time.Since(started); elapsed < tc.rtt/2 {
	114	+	t.Fatalf("client-to-target netem delay=%s, want at least %s", elapsed, tc.rtt/2)
	115	+	}
	116	+	}
	117	+	if tc.loss == 0 {
	118	+	// The relay learns the client from the first carriage packet. Replying
	119	+	// through that same relay address verifies the target-to-client half
	120	+	// of the configured RTT as well.
	121	+	started = time.Now()
	122	+	if _, err := target.WriteTo([]byte("reply"), relayAddr); err != nil {
	123	+	t.Fatal(err)
	124	+	}
	125	+	if err := client.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
	126	+	t.Fatal(err)
	127	+	}
	128	+	if _, _, err := client.ReadFrom(buf); err != nil {
	129	+	t.Fatalf("relay reply: %v", err)
	130	+	}
	131	+	if tc.rtt > 0 {
	132	+	if elapsed := time.Since(started); elapsed < tc.rtt/2 {
	133	+	t.Fatalf("target-to-client netem delay=%s, want at least %s", elapsed, tc.rtt/2)
	134	+	}
	135	+	}
	136	+	}
	137	+	// Keep the deadline short: extra carriage would prove the exact 1%%
	138	+	// loss cycle was not enforced.
	139	+	if err := target.SetReadDeadline(time.Now().Add(20 * time.Millisecond)); err != nil {
	140	+	t.Fatal(err)
	141	+	}
	142	+	if _, _, err := target.ReadFrom(buf); tc.want != tc.sent && err == nil {
	143	+	t.Fatalf("received more than %d packets with %d%% loss", tc.want, tc.loss)
	144	+	}
	145	+	})
	146	+	}
	147	+	}

M docs/performance.md +63 -91

@@ -1,107 +1,79 @@
1		-	# Performance methodology (M5)
2		-
3		-	This baseline measures in-process daemon work only. It includes coordinator-enabled
4		-	fixture scheduling metrics, but deliberately does not measure chunk
5		-	encoding, compression, GC tuning, or real network latency. Output metrics count frames and encoded output data produced
6		-	by the daemon; snapshot writes and serialized bytes are local snapshot-store
7		-	proxies. These metrics approximate local rendering, IPC payload, and persistence
8		-	work, not a Unix socket, SSH, or UDP transport. Real transport impairment and
9		-	latency benchmarks are deferred.
10		-
11		-	## Large-history daemon baseline
12		-
13		-	Use a single iteration only as a smoke check; it is not suitable for drawing
14		-	performance conclusions:
15		-
16		-	```sh
17		-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1x -benchmem
18		-	```
19		-
20		-	For before/after comparison, collect repeated samples for the full matrix
21		-	(all workloads and topologies), pinning CPU/governor and recording the host.
22		-	The benchmark harness must be committed and present in both measured
23		-	revisions: collect the baseline after the harness is introduced but before the
24		-	optimization, then collect the optimized revision with that same harness. The
25		-	fixed duration and count make the output suitable for `benchstat`:
26		-
27		-	```sh
28		-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1s -count=5 -benchmem > /tmp/vev-history-after.txt
29		-	benchstat /tmp/vev-history-before.txt /tmp/vev-history-after.txt
	1	+	# Performance measurement
	2	+
	3	+	## S5a transport observability
	4	+
	5	+	S5a is measurement-only. It records process-local runtime marks and harness-owned
	6	+	end-to-end boundaries without changing rendering, pacing, transport policy, wire
	7	+	bytes, or `ProtocolVersion`. The canonical matrix and workload/topology/
	8	+	transport coverage are in [`testdata/perf/manifest.json`](../testdata/perf/manifest.json):
	9	+	`1x4`, `4x1`, `4x4`, and `8x1`, all at `120x40` with 10,000 rows per pane;
	10	+	idle, active/all/inactive output, interactive flood, copy search, resize sweep,
	11	+	snapshot/output/resize, and attach/restore/tab switch; local, SSH stdio, UDP
	12	+	baseline, UDP 25/100 ms, and UDP 0%/1% loss. It has 252 scenarios (4×9×7),
	13	+	with inapplicable cases required to be explicit. The seven transport entries are
	14	+	`local`, `ssh_stdio`, `udp_baseline`, `udp_loss_0pct`, `udp_25ms`, `udp_100ms`,
	15	+	and `udp_loss_1pct`.
	16	+
	17	+	### Trace schema and clock domains
	18	+
	19	+	Every process JSONL record has exactly these fields:
	20	+	`schema` (uint), `process_id` (string), `component` (string), `scenario`
	21	+	(string), `run`, `sequence`, `request_id`, `epoch` (uint), `kind` (string),
	22	+	`tick` (int64), `bytes`, `fragments`, `retransmits`, `pending` (uint),
	23	+	`ack_rtt_nanos` (int64), and `valid` (bool). Schema 1 is represented by:
	24	+
	25	+	```json
	26	+	{"schema":1,"process_id":"p","component":"daemon","scenario":"s","run":1,"sequence":1,"request_id":1,"epoch":1,"kind":"diff_start","tick":0,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
30	27		```
31	28
32		-	The matrix exercises live paint, snapshot capture, copy-mode entry, copy search,
33		-	and resize at `120x40` client geometry. Each pane is populated with 10,000
34		-	full-width deterministic history rows containing a stable `needle-<tab>-<row>`
35		-	prefix and patterned text. The layouts are:
	29	+	`tick` is stamped only by the concrete JSONL observer using its process-local
	30	+	injected monotonic clock; producers supply no timestamp. The harness sets `VEV_PERF_TRACE`,
	31	+	`VEV_PERF_PROCESS_ID`, `VEV_PERF_SCENARIO`, and `VEV_PERF_RUN` for every
	32	+	launched role, identifying its exclusive trace path and process/scenario/run
	33	+	mapping. Components in one OS
	34	+	process share that observer. The harness clock is a separate monotonic domain
	35	+	and is the sole clock for input-injection → terminal-bytes-successfully-flushed
	36	+	end-to-end samples. Cross-process ticks are never ordered or subtracted;
	37	+	records correlate only by scenario/run/sequence/request/epoch IDs.
36	38
37		-	- 1 tab x 1 pane (control)
38		-	- 1 tab x 4 panes
39		-	- 4 tabs x 1 pane
40		-	- 4 tabs x 4 panes
41		-	- 8 tabs x 1 pane
	39	+	The named process-local spans are capture, compose, diff, queue wait,
	40	+	ACK-blocked interval, emit, adapter send, and adapter receive. Their start/end
	41	+	marks are paired within one process and summarized by component/adapter with
	42	+	raw records, counts, p50/p95/p99, and max. Diagnostic fields (bytes, fragments,
	43	+	retransmits, pending, ACK RTT) remain separate from duration calculations.
42	44
43		-	The fixture starts no PTY readers or transports. It primes the renderer shadow
44		-	before timing; live paint then alternates fixed writes through the coordinator.
45		-	The reported `outputframes/op` value counts output messages. `outputbytes/op`
46		-	counts their encoded terminal data and excludes the 24-byte output header;
47		-	`framepayloadbytes/op` counts the complete output payload including that header.
48		-	`snapshotwrites/op` and `snapshotbytes/op` are local snapshot-store writes and
49		-	serialized bytes. `coordinatorinvalidations/op`, `coordinatorwakes/op`, and
50		-	`coordinatorcoalesced/op` respectively count scheduling requests, delivered
51		-	wakes, and requests coalesced into those wakes; `coordinatorcoalescingratio`
52		-	is the average number of coalesced invalidations delivered per wake. They must
53		-	not be presented as network
54		-	throughput or latency.
	45	+	### Baseline procedure and validity
55	46
56		-	Capture a pre-change result outside the repository, including Go and kernel
57		-	context:
	47	+	Use the public CLI harness (not internal packages):
58	48
59	49		```sh
60		-	{
61		-	date -Is
62		-	go version
63		-	uname -a
64		-	git rev-parse HEAD
65		-	# Smoke only; do not use this result for comparisons.
66		-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1x -benchmem
67		-	# Repeated full matrix for every workload/topology used in conclusions.
68		-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1s -count=5 -benchmem
69		-	} > /tmp/vev-history-before.txt 2>&1
	50	+	go build -o /tmp/vev-perf ./
	51	+	rm -rf testdata/perf/results
	52	+	go run ./cmd/vev-perf-harness --vev-bin /tmp/vev-perf \
	53	+	--manifest testdata/perf/manifest.json --out testdata/perf/results \
	54	+	--warmup 10s --duration 30s --repetitions 10
70	55		```
71	56
72		-	## Immutable document allocation gate
	57	+	The minimums are 30 seconds measured and 10 independent repetitions; warmup is
	58	+	excluded. During the measured interval the harness injects at most one event per
	59	+	second. At least 10 complete event pairs are required: both injection and
	60	+	successful terminal flush must fall inside the measured interval; straddling or
	61	+	failed events are retained only as raw diagnostics and are not samples. Each run must contain `manifest.json`, `raw-harness.jsonl`, per-process
	62	+	JSONL, `runs.json`, and `summary.json`. Reject results with missing input/flush
	63	+	boundaries, unmatched IDs, bad correlation, nonmonotonic same-process sequence,
	64	+	negative same-domain spans, fewer than 10 complete in-interval event pairs,
	65	+	invalid denominators,
	66	+	any cross-process tick arithmetic, or unmet duration/repetition minima. Raw
	67	+	JSONL and run manifests are the evidence files; no result authorizes a policy
	68	+	or optimization change.
73	69
74		-	The copy-mode document retains an immutable view of existing scrollback rows and
75		-	clones only the visible frame. Visual-search models share that document; cloning
76		-	a model copies only its mutable input and match state. Check those two paths at
77		-	10,000 history rows with:
	70	+	## Existing in-process smoke checks
78	71
79		-	```sh
80		-	go test ./internal/usecase/copy ./internal/usecase/visualsearch -run '^$' -bench '^(BenchmarkNewSnapshot10KRows\|BenchmarkVisualSearchClone10KRows)$' -benchtime=1s -count=5 -benchmem
81		-	```
82		-
83		-	Treat `allocs/op` and `B/op` as allocation gates: compare the complete repeated
84		-	output with the accepted baseline before merging, and investigate any increase.
85		-	The automated scaling gates also compare equal row/match counts at narrow and
86		-	wide row widths; `B/op` may grow by at most 2x, a conservative relative
87		-	allowance that catches full cell-row copies without depending on an absolute
88		-	Go-version-specific budget. This scope does not yet include an
89		-	incremental/full-text search index: queries still scan the immutable document
90		-	and produce a fresh match list. Chunk encoding, compression, GC tuning, and real Unix-socket, SSH, or UDP
91		-	impairment/latency remain out of scope.
92		-
93		-	## Lower-level smoke baseline
94		-
95		-	Use this command to check renderer, VT, IPC, and daemon microbenchmarks without
96		-	claiming an end-to-end transport result:
	72	+	These commands measure local daemon work only and are not end-to-end transport
	73	+	results:
97	74
98	75		```sh
	76	+	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1x -benchmem
99	77		go test ./pkg/renderer ./pkg/vt ./internal/adapters/ipc -run '^$' -bench=. -benchmem
100	78		go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkComposeFloatingFrameCached$' -benchmem
101	79		```
102		-
103		-	Enable daemon pprof explicitly when collecting a daemon profile:
104		-
105		-	```sh
106		-	VEV_PPROF_ADDR=127.0.0.1:6060 vev --daemon
107		-	```

M internal/adapters/dgram/ssh_dialer.go +2 -1


@@ -102,6 +102,7 @@ type RemoteDialer struct {
102	102		BootstrapTimeout time.Duration
103	103		ProbeTimeout time.Duration
104	104		Log *slog.Logger
	105	+	RuntimeObserver ports.RuntimeObserver
105	106		}
106	107
107	108		func NewRemoteDialer(target, session string) RemoteDialer {

@@ -180,7 +181,7 @@ func (d RemoteDialer) Dial(ctx context.Context) (ports.Transport, error) {
180	181		RebindPacketConn: func(net.PacketConn) (net.PacketConn, error) {
181	182		return listenUDPPacket(ctx)
182	183		},
183		-	})
	184	+	}, WithRuntimeObserver(d.RuntimeObserver))
184	185		if err != nil {
185	186		_ = pc.Close()
186	187		return nil, udpUnavailable("create UDP transport", err, &stderr)

M internal/adapters/dgram/transport.go +96 -7


@@ -45,6 +45,14 @@ var (
45	45		errControlFull = errors.New("dgram: control queue full")
46	46		)
47	47
	48	+	type RuntimeOption func(*Transport)
	49	+
	50	+	// WithRuntimeObserver enables process-local marks without changing dgram's
	51	+	// behavior clock or carriage policy.
	52	+	func WithRuntimeObserver(observer ports.RuntimeObserver) RuntimeOption {
	53	+	return func(t *Transport) { t.runtimeObserver = observer }
	54	+	}
	55	+
48	56		type Options struct {
49	57		MTU int
50	58		ResendAfter time.Duration

@@ -168,6 +176,11 @@ type Transport struct {
168	176		// mutation and before ordered event publication.
169	177		afterLinkStateCommit func(ports.LinkState)
170	178		observe DiagnosticObserver
	179	+	runtimeObserver ports.RuntimeObserver
	180	+	operationMu sync.Mutex
	181	+	operationCount int
	182	+	closing bool
	183	+	operationsDone chan struct{}
171	184		diagnosticCh chan Diagnostic
172	185		heartbeat time.Duration
173	186		resendAfter time.Duration

@@ -311,11 +324,11 @@ func (s *writeDeadlineState) expire(now time.Time) {
311	324		s.mu.Unlock()
312	325		}
313	326
314		-	func NewTransport(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32) (*Transport, error) {
315		-	return NewTransportWithOptions(pc, peer, key, sendDir, recvDir, Options{})
	327	+	func NewTransport(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32, runtimeOpts ...RuntimeOption) (*Transport, error) {
	328	+	return NewTransportWithOptions(pc, peer, key, sendDir, recvDir, Options{}, runtimeOpts...)
316	329		}
317	330
318		-	func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32, opts Options) (*Transport, error) {
	331	+	func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32, opts Options, runtimeOpts ...RuntimeOption) (*Transport, error) {
319	332		c, err := pdgram.NewCodec(key)
320	333		if err != nil {
321	334		return nil, err

@@ -360,6 +373,11 @@ func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendD
360	373		nextRecvSeq: 1,
361	374		recvBuf: make(map[uint64]ports.Frame),
362	375		}
	376	+	for _, opt := range runtimeOpts {
	377	+	if opt != nil {
	378	+	opt(t)
	379	+	}
	380	+	}
363	381		t.sendWake = make(chan struct{})
364	382		t.control = make(chan controlRecord, controlQueueSize)
365	383		t.ackWake = make(chan struct{}, 1)

@@ -387,9 +405,19 @@ func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendD
387	405		return t, nil
388	406		}
389	407
390		-	func (t *Transport) Send(f ports.Frame) error { return t.send(f, false) }
	408	+	func (t *Transport) Send(f ports.Frame) error {
	409	+	end := t.beginRuntimeOperation(ports.RuntimeAdapterSendStart, uint64(len(f.Payload)))
	410	+	err := t.send(f, false)
	411	+	end(err == nil)
	412	+	return err
	413	+	}
391	414
392		-	func (t *Transport) SendAsync(f ports.Frame) error { return t.send(f, true) }
	415	+	func (t *Transport) SendAsync(f ports.Frame) error {
	416	+	end := t.beginRuntimeOperation(ports.RuntimeAdapterSendStart, uint64(len(f.Payload)))
	417	+	err := t.send(f, true)
	418	+	end(err == nil)
	419	+	return err
	420	+	}
393	421
394	422		// SendSynchronous owns queue reservation, fragment pacing, per-write deadlines,
395	423		// and close cancellation. Callers must not layer a shorter preflight timer over it.

@@ -568,25 +596,35 @@ func (t *Transport) removePendingLocked(seq uint64) bool {
568	596		}
569	597
570	598		func (t *Transport) Recv() (ports.Frame, error) {
	599	+	end := t.beginRuntimeOperation(ports.RuntimeAdapterReceiveStart, 0)
571	600		select {
572	601		case f := <-t.in:
	602	+	end(true)
573	603		return f, nil
574	604		case <-t.done:
575	605		t.mu.Lock()
576	606		err := t.closeErr
577	607		t.mu.Unlock()
578	608		if err != nil {
	609	+	end(false)
579	610		return ports.Frame{}, err
580	611		}
581		-	return ports.Frame{}, errors.New("dgram: closed")
	612	+	err = errors.New("dgram: closed")
	613	+	end(false)
	614	+	return ports.Frame{}, err
582	615		}
583	616		}
584	617		func (t *Transport) Close() error {
	618	+	done := t.beginShutdown()
585	619		t.closeWithError(errors.New("dgram: closed"))
586	620		t.mu.Lock()
587	621		pc := t.pc
588	622		t.mu.Unlock()
589		-	return pc.Close()
	623	+	err := pc.Close()
	624	+	if done != nil {
	625	+	<-done
	626	+	}
	627	+	return err
590	628		}
591	629
592	630		func (t *Transport) Peer() net.Addr { t.mu.Lock(); defer t.mu.Unlock(); return t.peer }

@@ -1585,4 +1623,55 @@ func decodeData(b []byte) (uint64, bool, ports.Frame, bool) {
1585	1623		}
1586	1624
1587	1625		// DatagramTransport marks Transport as a UDP-style datagram transport.
	1626	+	func (t *Transport) beginRuntimeOperation(start ports.RuntimeMarkKind, bytes uint64) func(bool) {
	1627	+	if t.runtimeObserver == nil {
	1628	+	return func(bool) {}
	1629	+	}
	1630	+	if !t.beginObservedOperation() {
	1631	+	return func(bool) {}
	1632	+	}
	1633	+	correlation := ports.NewRuntimeCorrelation()
	1634	+	t.runtimeObserver.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("dgram", correlation, start, bytes, true))
	1635	+	end := ports.RuntimeAdapterSendEnd
	1636	+	if start == ports.RuntimeAdapterReceiveStart {
	1637	+	end = ports.RuntimeAdapterReceiveEnd
	1638	+	}
	1639	+	return func(valid bool) {
	1640	+	defer t.finishObservedOperation()
	1641	+	t.runtimeObserver.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("dgram", correlation, end, bytes, valid))
	1642	+	}
	1643	+	}
	1644	+
	1645	+	func (t *Transport) beginObservedOperation() bool {
	1646	+	t.operationMu.Lock()
	1647	+	defer t.operationMu.Unlock()
	1648	+	if t.closing {
	1649	+	return false
	1650	+	}
	1651	+	if t.operationCount == 0 {
	1652	+	t.operationsDone = make(chan struct{})
	1653	+	}
	1654	+	t.operationCount++
	1655	+	return true
	1656	+	}
	1657	+
	1658	+	func (t *Transport) finishObservedOperation() {
	1659	+	t.operationMu.Lock()
	1660	+	defer t.operationMu.Unlock()
	1661	+	t.operationCount--
	1662	+	if t.operationCount == 0 {
	1663	+	close(t.operationsDone)
	1664	+	}
	1665	+	}
	1666	+
	1667	+	func (t *Transport) beginShutdown() <-chan struct{} {
	1668	+	t.operationMu.Lock()
	1669	+	defer t.operationMu.Unlock()
	1670	+	t.closing = true
	1671	+	if t.operationCount == 0 {
	1672	+	return nil
	1673	+	}
	1674	+	return t.operationsDone
	1675	+	}
	1676	+
1588	1677		func (*Transport) DatagramTransport() {}

A internal/adapters/dgram/transport_observability_test.go +202 -0

@@ -0,0 +1,202 @@
	1	+	package dgram
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"errors"
	6	+	"net"
	7	+	"os"
	8	+	"reflect"
	9	+	"strings"
	10	+	"sync"
	11	+	"testing"
	12	+	"time"
	13	+
	14	+	"github.com/bnema/vev/internal/adapters/observability"
	15	+	"github.com/bnema/vev/internal/ports"
	16	+	)
	17	+
	18	+	func TestTransportObservabilityDgramFailedCloseEndsReceiveAndSend(t *testing.T) {
	19	+	behaviorClock := newManualClock(time.Unix(0, 417))
	20	+	link := newSimulatedLink(behaviorClock, packetPolicy{})
	21	+	aPC, _ := newPairWithCapacity(link, 64)
	22	+	closeErr := errors.New("packet connection close failed")
	23	+	tracePath := t.TempDir() + "/dgram-close.jsonl"
	24	+	observer, closer, err := observability.NewJSONL(tracePath, dgramObserverClock{now: time.Unix(0, 521)}, "dgram-process")
	25	+	if err != nil {
	26	+	t.Fatalf("NewJSONL() error = %v", err)
	27	+	}
	28	+	receiveStarted := make(chan struct{})
	29	+	signalingObserver := &dgramStartObserver{RuntimeObserver: observer, receiveStarted: receiveStarted}
	30	+	transport, err := NewTransportWithOptions(&dgramCloseFailPC{fakePC: aPC, err: closeErr}, testAddr("b"), key(), 1, 2, Options{Clock: behaviorClock, ResendAfter: time.Hour, Heartbeat: time.Hour}, WithRuntimeObserver(signalingObserver))
	31	+	if err != nil {
	32	+	t.Fatalf("NewTransportWithOptions() error = %v", err)
	33	+	}
	34	+	if err := transport.Send(ports.Frame{Type: ports.MsgOutput, Payload: []byte("write failure")}); err == nil {
	35	+	t.Fatal("Send() error = nil on packet write failure")
	36	+	}
	37	+	recvDone := make(chan error, 1)
	38	+	go func() { _, err := transport.Recv(); recvDone <- err }()
	39	+	<-receiveStarted
	40	+	if err := transport.Close(); !errors.Is(err, closeErr) {
	41	+	t.Fatalf("Close() error = %v, want %v", err, closeErr)
	42	+	}
	43	+	if err := <-recvDone; err == nil {
	44	+	t.Fatal("Recv() error = nil after failed Close()")
	45	+	}
	46	+	if err := closer.Close(); err != nil {
	47	+	t.Fatalf("trace Close() error = %v", err)
	48	+	}
	49	+	assertDgramFailedSpanPairs(t, tracePath)
	50	+	}
	51	+
	52	+	type dgramStartObserver struct {
	53	+	ports.RuntimeObserver
	54	+	receiveStarted chan struct{}
	55	+	once sync.Once
	56	+	}
	57	+
	58	+	func (o *dgramStartObserver) ObserveRuntime(mark ports.RuntimeMark) {
	59	+	if mark.Kind == ports.RuntimeAdapterReceiveStart {
	60	+	o.once.Do(func() { close(o.receiveStarted) })
	61	+	}
	62	+	o.RuntimeObserver.ObserveRuntime(mark)
	63	+	}
	64	+
	65	+	type dgramCloseFailPC struct {
	66	+	*fakePC
	67	+	err error
	68	+	}
	69	+
	70	+	func (p *dgramCloseFailPC) WriteTo([]byte, net.Addr) (int, error) {
	71	+	return 0, errors.New("packet write failed")
	72	+	}
	73	+
	74	+	func (p *dgramCloseFailPC) Close() error {
	75	+	_ = p.fakePC.Close()
	76	+	return p.err
	77	+	}
	78	+
	79	+	func assertDgramFailedSpanPairs(t *testing.T, path string) {
	80	+	t.Helper()
	81	+	data, err := os.ReadFile(path)
	82	+	if err != nil {
	83	+	t.Fatalf("ReadFile(trace): %v", err)
	84	+	}
	85	+	type mark struct {
	86	+	ProcessID string `json:"process_id"`
	87	+	Scenario string `json:"scenario"`
	88	+	Run uint64 `json:"run"`
	89	+	Sequence uint64 `json:"sequence"`
	90	+	RequestID uint64 `json:"request_id"`
	91	+	Epoch uint64 `json:"epoch"`
	92	+	Kind string `json:"kind"`
	93	+	Valid bool `json:"valid"`
	94	+	}
	95	+	type spanKey struct {
	96	+	processID, scenario string
	97	+	run, sequence, requestID, epoch uint64
	98	+	kind string
	99	+	}
	100	+	starts := make(map[spanKey]mark)
	101	+	pairs := map[string]int{}
	102	+	for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
	103	+	var m mark
	104	+	if err := json.Unmarshal([]byte(line), &m); err != nil {
	105	+	t.Fatalf("Unmarshal(trace): %v", err)
	106	+	}
	107	+	key := spanKey{m.ProcessID, m.Scenario, m.Run, m.Sequence, m.RequestID, m.Epoch, m.Kind}
	108	+	switch m.Kind {
	109	+	case "adapter_receive_start", "adapter_send_start":
	110	+	if _, exists := starts[key]; exists {
	111	+	t.Fatalf("duplicate adapter start: %#v", m)
	112	+	}
	113	+	starts[key] = m
	114	+	case "adapter_receive_end", "adapter_send_end":
	115	+	key.kind = strings.TrimSuffix(m.Kind, "_end") + "_start"
	116	+	start, ok := starts[key]
	117	+	if !ok \|\| m.Valid {
	118	+	t.Fatalf("unmatched or valid failed end: start=%#v end=%#v", start, m)
	119	+	}
	120	+	pairs[strings.TrimSuffix(m.Kind, "_end")]++
	121	+	delete(starts, key)
	122	+	}
	123	+	}
	124	+	if len(starts) != 0 {
	125	+	t.Fatalf("unmatched adapter starts: %#v", starts)
	126	+	}
	127	+	if pairs["adapter_send"] != 1 \|\| pairs["adapter_receive"] != 1 \|\| len(pairs) != 2 {
	128	+	t.Fatalf("failed adapter span pairs=%v, want exactly one send and one receive", pairs)
	129	+	}
	130	+	}
	131	+
	132	+	func TestTransportObservabilityDgramKeepsBehaviorClockSeparate(t *testing.T) {
	133	+	behaviorClock := newManualClock(time.Unix(0, 401))
	134	+	link := newSimulatedLink(behaviorClock, packetPolicy{})
	135	+	aPC, bPC := newPairWithCapacity(link, 64)
	136	+	tracePath := t.TempDir() + "/dgram.jsonl"
	137	+	observer, closer, err := observability.NewJSONL(tracePath, dgramObserverClock{now: time.Unix(0, 509)}, "dgram-process")
	138	+	if err != nil {
	139	+	t.Fatalf("NewJSONL() error = %v", err)
	140	+	}
	141	+
	142	+	a, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{Clock: behaviorClock, ResendAfter: time.Hour, Heartbeat: time.Hour}, WithRuntimeObserver(observer))
	143	+	if err != nil {
	144	+	t.Fatalf("NewTransportWithOptions(a) error = %v", err)
	145	+	}
	146	+	defer func() { _ = a.Close() }()
	147	+	b, err := NewTransportWithOptions(bPC, testAddr("a"), key(), 2, 1, Options{Clock: behaviorClock, ResendAfter: time.Hour, Heartbeat: time.Hour}, WithRuntimeObserver(observer))
	148	+	if err != nil {
	149	+	t.Fatalf("NewTransportWithOptions(b) error = %v", err)
	150	+	}
	151	+	defer func() { _ = b.Close() }()
	152	+
	153	+	want := ports.Frame{Type: ports.MsgOutput, Payload: []byte("datagram carriage remains exact")}
	154	+	if err := a.Send(want); err != nil {
	155	+	t.Fatalf("Send() error = %v", err)
	156	+	}
	157	+	gotCh := make(chan ports.Frame, 1)
	158	+	errCh := make(chan error, 1)
	159	+	go func() {
	160	+	frame, recvErr := b.Recv()
	161	+	if recvErr != nil {
	162	+	errCh <- recvErr
	163	+	return
	164	+	}
	165	+	gotCh <- frame
	166	+	}()
	167	+	select {
	168	+	case err := <-errCh:
	169	+	t.Fatalf("Recv() error = %v", err)
	170	+	case got := <-gotCh:
	171	+	if !reflect.DeepEqual(got, want) {
	172	+	t.Fatalf("carried frame = %#v, want %#v", got, want)
	173	+	}
	174	+	case <-time.After(time.Second):
	175	+	t.Fatal("Recv() did not receive deterministic simulated datagram")
	176	+	}
	177	+	if err := closer.Close(); err != nil {
	178	+	t.Fatalf("trace Close() error = %v", err)
	179	+	}
	180	+	trace, err := os.ReadFile(tracePath)
	181	+	if err != nil {
	182	+	t.Fatalf("ReadFile(trace): %v", err)
	183	+	}
	184	+	for _, kind := range []string{"adapter_send_start", "adapter_send_end", "adapter_receive_start", "adapter_receive_end"} {
	185	+	if !strings.Contains(string(trace), `"kind":"`+kind+`"`) {
	186	+	t.Errorf("trace missing %s", kind)
	187	+	}
	188	+	}
	189	+	}
	190	+
	191	+	type dgramObserverClock struct{ now time.Time }
	192	+
	193	+	func (c dgramObserverClock) Now() time.Time { return c.now }
	194	+	func (c dgramObserverClock) NewTimer(d time.Duration) ports.Timer {
	195	+	return dgramObserverTimer{timer: time.NewTimer(d)}
	196	+	}
	197	+
	198	+	type dgramObserverTimer struct{ timer *time.Timer }
	199	+
	200	+	func (t dgramObserverTimer) C() <-chan time.Time { return t.timer.C }
	201	+	func (t dgramObserverTimer) Reset(d time.Duration) bool { return t.timer.Reset(d) }
	202	+	func (t dgramObserverTimer) Stop() bool { return t.timer.Stop() }

M internal/adapters/ipc/dial.go +7 -5

@@ -1,6 +1,7 @@
1	1		package ipc
2	2
3	3		import (
	4	+	"context"
4	5		"fmt"
5	6		"net"
6	7		"path/filepath"

@@ -8,15 +9,16 @@ import (
8	9		"github.com/bnema/vev/internal/ports"
9	10		)
10	11
11		-	// Dial connects to the daemon's socket in dir (dir/daemon.sock) and returns
12		-	// a Transport speaking vev's framed protocol. A dial failure (no daemon
	12	+	// DialContext connects to the daemon's socket in dir (dir/daemon.sock) and
	13	+	// returns a Transport speaking vev's framed protocol. A dial failure (no daemon
13	14		// listening, or a stale/absent socket) is reported as a wrapped error so
14	15		// callers can decide whether to spawn a daemon and retry.
15		-	func Dial(dir string) (ports.Transport, error) {
	16	+	func DialContext(ctx context.Context, dir string, opts ...Option) (ports.Transport, error) {
16	17		sockPath := filepath.Join(dir, socketFileName)
17		-	conn, err := net.Dial("unix", sockPath)
	18	+	var dialer net.Dialer
	19	+	conn, err := dialer.DialContext(ctx, "unix", sockPath)
18	20		if err != nil {
19	21		return nil, fmt.Errorf("ipc: dial %s: %w", sockPath, err)
20	22		}
21		-	return NewTransport(conn), nil
	23	+	return NewTransport(conn, opts...), nil
22	24		}

A internal/adapters/ipc/dial_test.go +79 -0

@@ -0,0 +1,79 @@
	1	+	package ipc
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"path/filepath"
	7	+	"testing"
	8	+	"time"
	9	+
	10	+	"github.com/bnema/vev/internal/ports"
	11	+	)
	12	+
	13	+	type dialTestObserver struct{}
	14	+
	15	+	func (dialTestObserver) ObserveRuntime(ports.RuntimeMark) {}
	16	+
	17	+	func TestDialContextCanceledOrExpired(t *testing.T) {
	18	+	tests := []struct {
	19	+	name string
	20	+	ctx func() (context.Context, context.CancelFunc)
	21	+	want error
	22	+	}{
	23	+	{
	24	+	name: "canceled",
	25	+	ctx: func() (context.Context, context.CancelFunc) {
	26	+	ctx, cancel := context.WithCancel(context.Background())
	27	+	cancel()
	28	+	return ctx, func() {}
	29	+	},
	30	+	want: context.Canceled,
	31	+	},
	32	+	{
	33	+	name: "deadline exceeded",
	34	+	ctx: func() (context.Context, context.CancelFunc) {
	35	+	return context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
	36	+	},
	37	+	want: context.DeadlineExceeded,
	38	+	},
	39	+	}
	40	+
	41	+	for _, tt := range tests {
	42	+	t.Run(tt.name, func(t *testing.T) {
	43	+	ctx, cancel := tt.ctx()
	44	+	defer cancel()
	45	+
	46	+	transport, err := DialContext(ctx, t.TempDir())
	47	+	if transport != nil {
	48	+	t.Fatal("DialContext returned a transport for a completed context")
	49	+	}
	50	+	if !errors.Is(err, tt.want) {
	51	+	t.Fatalf("DialContext error = %v, want wrapping %v", err, tt.want)
	52	+	}
	53	+	})
	54	+	}
	55	+	}
	56	+
	57	+	func TestDialContextPreservesOptions(t *testing.T) {
	58	+	dir := filepath.Join(t.TempDir(), "vev")
	59	+	listener, err := Listen(dir)
	60	+	if err != nil {
	61	+	t.Fatalf("Listen: %v", err)
	62	+	}
	63	+	defer func() { _ = listener.Close() }()
	64	+
	65	+	observer := dialTestObserver{}
	66	+	transport, err := DialContext(context.Background(), dir, WithRuntimeObserver(observer))
	67	+	if err != nil {
	68	+	t.Fatalf("DialContext: %v", err)
	69	+	}
	70	+	defer func() { _ = transport.Close() }()
	71	+
	72	+	got, ok := transport.(*unixTransport)
	73	+	if !ok {
	74	+	t.Fatalf("DialContext transport = %T, want *unixTransport", transport)
	75	+	}
	76	+	if got.observer != observer {
	77	+	t.Fatal("DialContext did not apply runtime observer option")
	78	+	}
	79	+	}

M internal/adapters/ipc/listener.go +4 -3


@@ -25,6 +25,7 @@ var ErrDaemonRunning = errors.New("ipc: a daemon is already listening on this so
25	25		type unixListener struct {
26	26		ln *net.UnixListener
27	27		addr string
	28	+	opts []Option
28	29		}
29	30
30	31		// Listen creates the socket directory (0700) if needed and starts

@@ -35,7 +36,7 @@ type unixListener struct {
35	36		// the previous owner died without cleaning up, so the stale socket file is
36	37		// unlinked and bind is retried once. A successful dial means a live daemon
37	38		// owns the socket, and Listen returns ErrDaemonRunning.
38		-	func Listen(dir string) (ports.Listener, error) {
	39	+	func Listen(dir string, opts ...Option) (ports.Listener, error) {
39	40		if err := safedir.EnsurePrivate(dir); err != nil {
40	41		return nil, fmt.Errorf("ipc: securing socket directory: %w", err)
41	42		}

@@ -69,7 +70,7 @@ func Listen(dir string) (ports.Listener, error) {
69	70		return nil, fmt.Errorf("ipc: chmod %s: %w", sockPath, err)
70	71		}
71	72
72		-	return &unixListener{ln: ln, addr: sockPath}, nil
	73	+	return &unixListener{ln: ln, addr: sockPath, opts: opts}, nil
73	74		}
74	75
75	76		// bindUnix binds and listens on sockPath.

@@ -104,7 +105,7 @@ func (l *unixListener) Accept() (ports.Transport, error) {
104	105		if err != nil {
105	106		return nil, err
106	107		}
107		-	return NewTransport(conn), nil
	108	+	return NewTransport(conn, l.opts...), nil
108	109		}
109	110
110	111		// Close stops accepting connections. The underlying net.UnixListener

M internal/adapters/ipc/transport.go +91 -5


@@ -5,6 +5,7 @@ import (
5	5		"errors"
6	6		"io"
7	7		"net"
	8	+	"sync"
8	9
9	10		"github.com/bnema/vev/internal/ports"
10	11		)

@@ -35,8 +36,21 @@ var ErrFrameTooLarge = errors.New("ipc: frame exceeds maximum length")
35	36		// goroutine and a single writer goroutine); it is not safe for concurrent
36	37		// calls to Send from multiple goroutines, nor Recv from multiple
37	38		// goroutines.
	39	+	type Option func(*unixTransport)
	40	+
	41	+	// WithRuntimeObserver enables process-local transport marks. It accepts no
	42	+	// clock: timestamp ownership belongs to the configured observer.
	43	+	func WithRuntimeObserver(observer ports.RuntimeObserver) Option {
	44	+	return func(t *unixTransport) { t.observer = observer }
	45	+	}
	46	+
38	47		type unixTransport struct {
39		-	conn net.Conn
	48	+	conn net.Conn
	49	+	observer ports.RuntimeObserver
	50	+	operationMu sync.Mutex
	51	+	operationCount int
	52	+	closing bool
	53	+	operationsDone chan struct{}
40	54
41	55		// readBuf is reused across Recv calls (grow-once strategy): it grows
42	56		// to fit the largest frame seen so far and is never shrunk. The Frame

@@ -47,16 +61,24 @@ type unixTransport struct {
47	61
48	62		// NewTransport wraps conn as a ports.Transport speaking vev's framed
49	63		// binary protocol.
50		-	func NewTransport(conn net.Conn) ports.Transport {
51		-	return &unixTransport{conn: conn}
	64	+	func NewTransport(conn net.Conn, opts ...Option) ports.Transport {
	65	+	t := &unixTransport{conn: conn}
	66	+	for _, opt := range opts {
	67	+	if opt != nil {
	68	+	opt(t)
	69	+	}
	70	+	}
	71	+	return t
52	72		}
53	73
54	74		// Send writes f as a single frame: a 4-byte big-endian length (covering
55	75		// the type byte and payload), the type byte, then the payload — assembled
56	76		// into one buffer and written with a single Write call.
57	77		func (t *unixTransport) Send(f ports.Frame) error {
	78	+	end := t.beginOperation(ports.RuntimeAdapterSendStart, uint64(len(f.Payload)))
58	79		n := 1 + len(f.Payload) // type + payload
59	80		if n > maxFrameLen {
	81	+	end(false)
60	82		return ErrFrameTooLarge
61	83		}
62	84

@@ -66,6 +88,7 @@ func (t *unixTransport) Send(f ports.Frame) error {
66	88		copy(buf[frameHeaderLen+1:], f.Payload)
67	89
68	90		_, err := t.conn.Write(buf)
	91	+	end(err == nil)
69	92		return err
70	93		}
71	94

@@ -73,16 +96,20 @@ func (t *unixTransport) Send(f ports.Frame) error {
73	96		// (type byte + payload). It blocks until a full frame arrives, the
74	97		// connection is closed (io.EOF), or an error occurs.
75	98		func (t *unixTransport) Recv() (ports.Frame, error) {
	99	+	end := t.beginOperation(ports.RuntimeAdapterReceiveStart, 0)
76	100		var hdr [frameHeaderLen]byte
77	101		if _, err := io.ReadFull(t.conn, hdr[:]); err != nil {
	102	+	end(false)
78	103		return ports.Frame{}, err
79	104		}
80	105
81	106		n := binary.BigEndian.Uint32(hdr[:])
82	107		if n == 0 {
	108	+	end(false)
83	109		return ports.Frame{}, ErrZeroLengthFrame
84	110		}
85	111		if n > maxFrameLen {
	112	+	end(false)
86	113		return ports.Frame{}, ErrFrameTooLarge
87	114		}
88	115

@@ -92,6 +119,7 @@ func (t *unixTransport) Recv() (ports.Frame, error) {
92	119		t.readBuf = t.readBuf[:n]
93	120		}
94	121		if _, err := io.ReadFull(t.conn, t.readBuf); err != nil {
	122	+	end(false)
95	123		return ports.Frame{}, err
96	124		}
97	125

@@ -100,10 +128,68 @@ func (t *unixTransport) Recv() (ports.Frame, error) {
100	128		payload = make([]byte, n-1)
101	129		copy(payload, t.readBuf[1:])
102	130		}
	131	+	end(true)
103	132		return ports.Frame{Type: ports.MsgType(t.readBuf[0]), Payload: payload}, nil
104	133		}
105	134
106		-	// Close closes the underlying connection.
	135	+	func (t *unixTransport) beginOperation(start ports.RuntimeMarkKind, bytes uint64) func(bool) {
	136	+	if t.observer == nil {
	137	+	return func(bool) {}
	138	+	}
	139	+	if !t.beginObservedOperation() {
	140	+	return func(bool) {}
	141	+	}
	142	+	correlation := ports.NewRuntimeCorrelation()
	143	+	t.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("ipc", correlation, start, bytes, true))
	144	+	end := ports.RuntimeAdapterSendEnd
	145	+	if start == ports.RuntimeAdapterReceiveStart {
	146	+	end = ports.RuntimeAdapterReceiveEnd
	147	+	}
	148	+	return func(valid bool) {
	149	+	defer t.finishObservedOperation()
	150	+	t.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("ipc", correlation, end, bytes, valid))
	151	+	}
	152	+	}
	153	+
	154	+	func (t *unixTransport) beginObservedOperation() bool {
	155	+	t.operationMu.Lock()
	156	+	defer t.operationMu.Unlock()
	157	+	if t.closing {
	158	+	return false
	159	+	}
	160	+	if t.operationCount == 0 {
	161	+	t.operationsDone = make(chan struct{})
	162	+	}
	163	+	t.operationCount++
	164	+	return true
	165	+	}
	166	+
	167	+	func (t *unixTransport) finishObservedOperation() {
	168	+	t.operationMu.Lock()
	169	+	defer t.operationMu.Unlock()
	170	+	t.operationCount--
	171	+	if t.operationCount == 0 {
	172	+	close(t.operationsDone)
	173	+	}
	174	+	}
	175	+
	176	+	func (t *unixTransport) beginShutdown() <-chan struct{} {
	177	+	t.operationMu.Lock()
	178	+	defer t.operationMu.Unlock()
	179	+	t.closing = true
	180	+	if t.operationCount == 0 {
	181	+	return nil
	182	+	}
	183	+	return t.operationsDone
	184	+	}
	185	+
	186	+	// Close closes the underlying connection and waits for any interrupted
	187	+	// carriage operation to write its matching failed end mark.
107	188		func (t *unixTransport) Close() error {
108		-	return t.conn.Close()
	189	+	done := t.beginShutdown()
	190	+	err := t.conn.Close()
	191	+	if done != nil {
	192	+	<-done
	193	+	}
	194	+	return err
109	195		}

A internal/adapters/ipc/transport_observability_test.go +191 -0

@@ -0,0 +1,191 @@
	1	+	package ipc
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"net"
	6	+	"os"
	7	+	"reflect"
	8	+	"strings"
	9	+	"sync"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"github.com/bnema/vev/internal/adapters/observability"
	14	+	"github.com/bnema/vev/internal/ports"
	15	+	)
	16	+
	17	+	func TestTransportObservabilityIPCEOFAndCloseEndFailedSpans(t *testing.T) {
	18	+	trace := t.TempDir() + "/ipc-shutdown.jsonl"
	19	+	observer, closer, err := observability.NewJSONL(trace, ipcRuntimeClock{now: time.Unix(0, 223)}, "ipc-process")
	20	+	if err != nil {
	21	+	t.Fatalf("NewJSONL() error = %v", err)
	22	+	}
	23	+
	24	+	// EOF from the peer and a locally closed blocked receive are distinct
	25	+	// shutdown paths; both must record a failed, correlated end.
	26	+	for _, closeLocal := range []bool{false, true} {
	27	+	left, right := net.Pipe()
	28	+	var started <-chan struct{}
	29	+	conn := left
	30	+	if closeLocal {
	31	+	observedConn := &ipcReadStartedConn{Conn: left, started: make(chan struct{})}
	32	+	conn, started = observedConn, observedConn.started
	33	+	}
	34	+	transport := NewTransport(conn, WithRuntimeObserver(observer))
	35	+	recvDone := make(chan error, 1)
	36	+	go func() { _, err := transport.Recv(); recvDone <- err }()
	37	+	if closeLocal {
	38	+	<-started
	39	+	if err := transport.Close(); err != nil {
	40	+	t.Fatalf("Close() error = %v", err)
	41	+	}
	42	+	} else if err := right.Close(); err != nil {
	43	+	t.Fatalf("peer Close() error = %v", err)
	44	+	}
	45	+	if err := <-recvDone; err == nil {
	46	+	t.Fatal("Recv() error = nil after transport shutdown")
	47	+	}
	48	+	_ = left.Close()
	49	+	_ = right.Close()
	50	+	}
	51	+
	52	+	// A peer shutdown also makes Send fail after its start mark.
	53	+	left, right := net.Pipe()
	54	+	transport := NewTransport(left, WithRuntimeObserver(observer))
	55	+	_ = right.Close()
	56	+	if err := transport.Send(ports.Frame{Type: ports.MsgOutput, Payload: []byte("failed")}); err == nil {
	57	+	t.Fatal("Send() error = nil after peer close")
	58	+	}
	59	+	_ = transport.Close()
	60	+	if err := closer.Close(); err != nil {
	61	+	t.Fatalf("trace Close() error = %v", err)
	62	+	}
	63	+	assertIPCFailedSpanPairs(t, trace)
	64	+	}
	65	+
	66	+	type ipcReadStartedConn struct {
	67	+	net.Conn
	68	+	started chan struct{}
	69	+	once sync.Once
	70	+	}
	71	+
	72	+	func (c *ipcReadStartedConn) Read(p []byte) (int, error) {
	73	+	c.once.Do(func() { close(c.started) })
	74	+	return c.Conn.Read(p)
	75	+	}
	76	+
	77	+	func assertIPCFailedSpanPairs(t *testing.T, path string) {
	78	+	t.Helper()
	79	+	data, err := os.ReadFile(path)
	80	+	if err != nil {
	81	+	t.Fatalf("ReadFile(trace): %v", err)
	82	+	}
	83	+	type mark struct {
	84	+	ProcessID string `json:"process_id"`
	85	+	Component string `json:"component"`
	86	+	Scenario string `json:"scenario"`
	87	+	Run uint64 `json:"run"`
	88	+	Sequence uint64 `json:"sequence"`
	89	+	RequestID uint64 `json:"request_id"`
	90	+	Epoch uint64 `json:"epoch"`
	91	+	Kind string `json:"kind"`
	92	+	Valid bool `json:"valid"`
	93	+	}
	94	+	type spanKey struct {
	95	+	processID, scenario string
	96	+	run, sequence, requestID, epoch uint64
	97	+	}
	98	+	starts := make(map[spanKey]mark)
	99	+	startCounts := map[string]int{}
	100	+	for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
	101	+	var m mark
	102	+	if err := json.Unmarshal([]byte(line), &m); err != nil {
	103	+	t.Fatalf("Unmarshal(trace): %v", err)
	104	+	}
	105	+	key := spanKey{m.ProcessID, m.Scenario, m.Run, m.Sequence, m.RequestID, m.Epoch}
	106	+	switch m.Kind {
	107	+	case "adapter_receive_start", "adapter_send_start":
	108	+	if m.ProcessID != "ipc-process" \|\| m.Component != "ipc" \|\| m.Scenario != "runtime" \|\| m.Run != 1 \|\| m.Sequence == 0 \|\| m.RequestID != m.Sequence \|\| m.Epoch != m.Sequence \|\| !m.Valid {
	109	+	t.Fatalf("adapter start has invalid identity: %#v", m)
	110	+	}
	111	+	if _, exists := starts[key]; exists {
	112	+	t.Fatalf("duplicate adapter start for %#v", key)
	113	+	}
	114	+	starts[key] = m
	115	+	startCounts[m.Kind]++
	116	+	case "adapter_receive_end", "adapter_send_end":
	117	+	start, ok := starts[key]
	118	+	if !ok \|\| strings.TrimSuffix(start.Kind, "_start") != strings.TrimSuffix(m.Kind, "_end") {
	119	+	t.Fatalf("unmatched adapter end %#v", m)
	120	+	}
	121	+	if m.Valid {
	122	+	t.Fatalf("failed adapter end is valid: %#v", m)
	123	+	}
	124	+	delete(starts, key)
	125	+	}
	126	+	}
	127	+	if len(starts) != 0 {
	128	+	t.Fatalf("unmatched adapter starts: %#v", starts)
	129	+	}
	130	+	if got := startCounts["adapter_receive_start"]; got != 2 {
	131	+	t.Errorf("adapter receive starts = %d, want 2", got)
	132	+	}
	133	+	if got := startCounts["adapter_send_start"]; got != 1 {
	134	+	t.Errorf("adapter send starts = %d, want 1", got)
	135	+	}
	136	+	}
	137	+
	138	+	func TestTransportObservabilityIPCMarksCarriageWithoutChangingBytes(t *testing.T) {
	139	+	trace := t.TempDir() + "/ipc.jsonl"
	140	+	observer, closer, err := observability.NewJSONL(trace, ipcRuntimeClock{now: time.Unix(0, 211)}, "ipc-process")
	141	+	if err != nil {
	142	+	t.Fatalf("NewJSONL() error = %v", err)
	143	+	}
	144	+
	145	+	left, right := net.Pipe()
	146	+	defer func() { _ = left.Close() }()
	147	+	defer func() { _ = right.Close() }()
	148	+	client := NewTransport(left, WithRuntimeObserver(observer))
	149	+	server := NewTransport(right, WithRuntimeObserver(observer))
	150	+	want := ports.Frame{Type: ports.MsgOutput, Payload: []byte("wire bytes stay exact")}
	151	+
	152	+	var wg sync.WaitGroup
	153	+	wg.Go(func() {
	154	+	if err := client.Send(want); err != nil {
	155	+	t.Errorf("Send() error = %v", err)
	156	+	}
	157	+	})
	158	+	got, err := server.Recv()
	159	+	wg.Wait()
	160	+	if err != nil {
	161	+	t.Fatalf("Recv() error = %v", err)
	162	+	}
	163	+	if !reflect.DeepEqual(got, want) {
	164	+	t.Fatalf("carried frame = %#v, want %#v", got, want)
	165	+	}
	166	+	if err := closer.Close(); err != nil {
	167	+	t.Fatalf("trace Close() error = %v", err)
	168	+	}
	169	+	traceBytes, err := os.ReadFile(trace)
	170	+	if err != nil {
	171	+	t.Fatalf("ReadFile(trace): %v", err)
	172	+	}
	173	+	for _, kind := range []string{"adapter_send_start", "adapter_send_end", "adapter_receive_start", "adapter_receive_end"} {
	174	+	if !strings.Contains(string(traceBytes), `"kind":"`+kind+`"`) {
	175	+	t.Errorf("trace missing %s", kind)
	176	+	}
	177	+	}
	178	+	}
	179	+
	180	+	type ipcRuntimeClock struct{ now time.Time }
	181	+
	182	+	func (c ipcRuntimeClock) Now() time.Time { return c.now }
	183	+	func (c ipcRuntimeClock) NewTimer(d time.Duration) ports.Timer {
	184	+	return ipcRuntimeTimer{timer: time.NewTimer(d)}
	185	+	}
	186	+
	187	+	type ipcRuntimeTimer struct{ timer *time.Timer }
	188	+
	189	+	func (t ipcRuntimeTimer) C() <-chan time.Time { return t.timer.C }
	190	+	func (t ipcRuntimeTimer) Reset(d time.Duration) bool { return t.timer.Reset(d) }
	191	+	func (t ipcRuntimeTimer) Stop() bool { return t.timer.Stop() }

A internal/adapters/observability/jsonl.go +100 -0

@@ -0,0 +1,100 @@
	1	+	// Package observability contains concrete process-local trace sinks.
	2	+	package observability
	3	+
	4	+	import (
	5	+	"encoding/json"
	6	+	"io"
	7	+	"os"
	8	+	"sync"
	9	+
	10	+	"github.com/bnema/vev/internal/ports"
	11	+	)
	12	+
	13	+	// NewJSONL opens path as the sole timestamp owner for one OS process. Marks
	14	+	// carrying a caller timestamp are rejected rather than rewritten, preventing
	15	+	// accidental arithmetic across clock domains.
	16	+	func NewJSONL(path string, clock ports.Clock, processID string) (ports.RuntimeObserver, io.Closer, error) {
	17	+	if path == "" \|\| clock == nil \|\| processID == "" {
	18	+	return nil, nil, os.ErrInvalid
	19	+	}
	20	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_APPEND, 0o600)
	21	+	if err != nil {
	22	+	return nil, nil, err
	23	+	}
	24	+	j := &jsonl{file: f, clock: clock, processID: processID}
	25	+	return j, j, nil
	26	+	}
	27	+
	28	+	type jsonl struct {
	29	+	mu sync.Mutex
	30	+	file *os.File
	31	+	clock ports.Clock
	32	+	processID string
	33	+	closed bool
	34	+	err error
	35	+	}
	36	+
	37	+	type record struct {
	38	+	Schema uint16 `json:"schema"`
	39	+	ProcessID string `json:"process_id"`
	40	+	Component string `json:"component"`
	41	+	Scenario string `json:"scenario"`
	42	+	Run uint64 `json:"run"`
	43	+	Sequence uint64 `json:"sequence"`
	44	+	RequestID uint64 `json:"request_id"`
	45	+	Epoch uint64 `json:"epoch"`
	46	+	Kind ports.RuntimeMarkKind `json:"kind"`
	47	+	Tick int64 `json:"tick"`
	48	+	Bytes uint64 `json:"bytes"`
	49	+	Fragments uint64 `json:"fragments"`
	50	+	Retransmits uint64 `json:"retransmits"`
	51	+	Pending uint64 `json:"pending"`
	52	+	AckRTTNanos int64 `json:"ack_rtt_nanos"`
	53	+	Valid bool `json:"valid"`
	54	+	}
	55	+
	56	+	func (j *jsonl) ObserveRuntime(mark ports.RuntimeMark) {
	57	+	// Tick is a write-once value owned by this adapter. Reject it before any
	58	+	// mutation so callers cannot smuggle a timestamp from their own clock.
	59	+	if j == nil \|\| mark.Tick != 0 {
	60	+	return
	61	+	}
	62	+	mark.ProcessID = j.processID
	63	+	if !ports.RuntimeMarkValid(mark) {
	64	+	return
	65	+	}
	66	+	j.mu.Lock()
	67	+	defer j.mu.Unlock()
	68	+	if j.closed {
	69	+	return
	70	+	}
	71	+	r := record{Schema: mark.Schema, ProcessID: mark.ProcessID, Component: mark.Component, Scenario: mark.Scenario, Run: mark.Run, Sequence: mark.Sequence, RequestID: mark.RequestID, Epoch: mark.Epoch, Kind: mark.Kind, Tick: j.clock.Now().UnixNano(), Bytes: mark.Bytes, Fragments: mark.Fragments, Retransmits: mark.Retransmits, Pending: mark.Pending, AckRTTNanos: mark.AckRTTNanos, Valid: mark.Valid}
	72	+	encoded, err := json.Marshal(r)
	73	+	if err != nil {
	74	+	j.recordError(err)
	75	+	return
	76	+	}
	77	+	if _, err := j.file.Write(append(encoded, '\n')); err != nil {
	78	+	j.recordError(err)
	79	+	}
	80	+	}
	81	+
	82	+	func (j *jsonl) recordError(err error) {
	83	+	if err != nil && j.err == nil {
	84	+	j.err = err
	85	+	}
	86	+	}
	87	+
	88	+	func (j *jsonl) Close() error {
	89	+	j.mu.Lock()
	90	+	defer j.mu.Unlock()
	91	+	if j.closed {
	92	+	return nil
	93	+	}
	94	+	j.closed = true
	95	+	closeErr := j.file.Close()
	96	+	if j.err != nil {
	97	+	return j.err
	98	+	}
	99	+	return closeErr
	100	+	}

A internal/adapters/observability/jsonl_test.go +193 -0

@@ -0,0 +1,193 @@
	1	+	package observability
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/json"
	6	+	"os"
	7	+	"sync"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/vev/internal/ports"
	12	+	)
	13	+
	14	+	func TestTransportObservabilityJSONLOwnsTicksAndSchema(t *testing.T) {
	15	+	path := t.TempDir() + "/daemon.jsonl"
	16	+	observer, closer, err := NewJSONL(path, fixedRuntimeClock{now: time.Unix(0, 41)}, "daemon-process")
	17	+	if err != nil {
	18	+	t.Fatalf("NewJSONL() error = %v", err)
	19	+	}
	20	+
	21	+	observer.ObserveRuntime(validRuntimeMark(ports.RuntimeComposeStart))
	22	+	// A producer's clock domain must never enter a JSONL record. Rejection is
	23	+	// observable as no second record, rather than a silently rewritten mark.
	24	+	producerTicked := validRuntimeMark(ports.RuntimeComposeEnd)
	25	+	producerTicked.Tick = 99
	26	+	observer.ObserveRuntime(producerTicked)
	27	+	if err := closer.Close(); err != nil {
	28	+	t.Fatalf("Close() error = %v", err)
	29	+	}
	30	+
	31	+	records := readJSONLRecords(t, path)
	32	+	if len(records) != 1 {
	33	+	t.Fatalf("JSONL records = %d, want 1 (caller Tick must be rejected)", len(records))
	34	+	}
	35	+	wantKeys := map[string]bool{
	36	+	"schema": true, "process_id": true, "component": true, "scenario": true,
	37	+	"run": true, "sequence": true, "request_id": true, "epoch": true,
	38	+	"kind": true, "tick": true, "bytes": true, "fragments": true,
	39	+	"retransmits": true, "pending": true, "ack_rtt_nanos": true, "valid": true,
	40	+	}
	41	+	if len(records[0]) != len(wantKeys) {
	42	+	t.Fatalf("JSONL fields = %#v, want exactly canonical schema", records[0])
	43	+	}
	44	+	for key := range wantKeys {
	45	+	if _, ok := records[0][key]; !ok {
	46	+	t.Errorf("JSONL missing %q", key)
	47	+	}
	48	+	}
	49	+	if got := records[0]["tick"]; got != float64(41) {
	50	+	t.Errorf("JSONL tick = %v, want observer clock tick 41", got)
	51	+	}
	52	+	if got := records[0]["process_id"]; got != "daemon-process" {
	53	+	t.Errorf("JSONL process_id = %v, want observer process", got)
	54	+	}
	55	+	}
	56	+
	57	+	func TestTransportObservabilityJSONLRejectsInvalidRequiredFields(t *testing.T) {
	58	+	for _, mark := range []ports.RuntimeMark{
	59	+	{},
	60	+	{Schema: 1, ProcessID: "p", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, Kind: ports.RuntimeEmitStart},
	61	+	{Schema: 2, ProcessID: "p", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, Kind: ports.RuntimeEmitStart, Valid: true},
	62	+	} {
	63	+	path := t.TempDir() + "/invalid.jsonl"
	64	+	observer, closer, err := NewJSONL(path, fixedRuntimeClock{now: time.Unix(0, 5)}, "p")
	65	+	if err != nil {
	66	+	t.Fatalf("NewJSONL() error = %v", err)
	67	+	}
	68	+	observer.ObserveRuntime(mark)
	69	+	if err := closer.Close(); err != nil {
	70	+	t.Fatalf("Close() error = %v", err)
	71	+	}
	72	+	if records := readJSONLRecords(t, path); len(records) != 0 {
	73	+	t.Errorf("invalid mark %#v wrote %#v", mark, records)
	74	+	}
	75	+	}
	76	+	}
	77	+
	78	+	func TestTransportObservabilityJSONLWriteErrorSurfacesOnClose(t *testing.T) {
	79	+	// /dev/full accepts Open but rejects every write, exercising the concrete
	80	+	// observer lifecycle without changing the one-way RuntimeObserver port.
	81	+	observer, closer, err := NewJSONL("/dev/full", fixedRuntimeClock{now: time.Unix(0, 1)}, "p")
	82	+	if err != nil {
	83	+	t.Skipf("/dev/full unavailable: %v", err)
	84	+	}
	85	+	observer.ObserveRuntime(validRuntimeMark(ports.RuntimeEmitStart))
	86	+	if err := closer.Close(); err == nil {
	87	+	t.Fatal("Close() error = nil, want surfaced JSONL write error")
	88	+	}
	89	+	}
	90	+
	91	+	func TestTransportObservabilitySameProcessSpanClockDomains(t *testing.T) {
	92	+	for _, tc := range []struct {
	93	+	name string
	94	+	start ports.RuntimeMarkKind
	95	+	end ports.RuntimeMarkKind
	96	+	field string
	97	+	}{
	98	+	{"diff", ports.RuntimeDiffStart, ports.RuntimeDiffEnd, "diff_duration"},
	99	+	{"queue", ports.RuntimeQueueEnqueued, ports.RuntimeQueueDequeued, "queue_wait"},
	100	+	{"ack", ports.RuntimeACKBlockedStart, ports.RuntimeACKBlockedEnd, "ack_blocked_interval"},
	101	+	{"send", ports.RuntimeAdapterSendStart, ports.RuntimeAdapterSendEnd, "adapter_send_duration"},
	102	+	{"receive", ports.RuntimeAdapterReceiveStart, ports.RuntimeAdapterReceiveEnd, "adapter_receive_duration"},
	103	+	} {
	104	+	t.Run(tc.name, func(t *testing.T) {
	105	+	start := validRuntimeMark(tc.start)
	106	+	start.Tick = 11
	107	+	end := start
	108	+	end.Kind, end.Tick = tc.end, 19
	109	+	if got, err := sameProcessDuration(start, end); err != nil \|\| got != 8 {
	110	+	t.Fatalf("%s = %d, %v; want 8, nil", tc.field, got, err)
	111	+	}
	112	+
	113	+	end.ProcessID = "other-process"
	114	+	if _, err := sameProcessDuration(start, end); err == nil {
	115	+	t.Fatalf("%s accepted cross-process ticks", tc.field)
	116	+	}
	117	+	})
	118	+	}
	119	+	}
	120	+
	121	+	func TestTransportObservabilityJSONLSerializesConcurrentMarks(t *testing.T) {
	122	+	path := t.TempDir() + "/concurrent.jsonl"
	123	+	observer, closer, err := NewJSONL(path, fixedRuntimeClock{now: time.Unix(0, 73)}, "ipc-process")
	124	+	if err != nil {
	125	+	t.Fatalf("NewJSONL() error = %v", err)
	126	+	}
	127	+	var wg sync.WaitGroup
	128	+	for i := uint64(0); i < 32; i++ {
	129	+	wg.Add(1)
	130	+	go func(sequence uint64) {
	131	+	defer wg.Done()
	132	+	mark := validRuntimeMark(ports.RuntimeAdapterSendEnd)
	133	+	mark.Sequence = sequence + 1
	134	+	observer.ObserveRuntime(mark)
	135	+	}(i)
	136	+	}
	137	+	wg.Wait()
	138	+	if err := closer.Close(); err != nil {
	139	+	t.Fatalf("Close() error = %v", err)
	140	+	}
	141	+	if records := readJSONLRecords(t, path); len(records) != 32 {
	142	+	t.Fatalf("concurrently serialized records = %d, want 32", len(records))
	143	+	}
	144	+	}
	145	+
	146	+	func sameProcessDuration(start, end ports.RuntimeMark) (int64, error) {
	147	+	if start.ProcessID == "" \|\| start.ProcessID != end.ProcessID {
	148	+	return 0, os.ErrInvalid
	149	+	}
	150	+	if start.Scenario != end.Scenario \|\| start.Run != end.Run \|\| start.Sequence != end.Sequence \|\| start.RequestID != end.RequestID \|\| start.Epoch != end.Epoch \|\| end.Tick < start.Tick {
	151	+	return 0, os.ErrInvalid
	152	+	}
	153	+	return end.Tick - start.Tick, nil
	154	+	}
	155	+
	156	+	func validRuntimeMark(kind ports.RuntimeMarkKind) ports.RuntimeMark {
	157	+	return ports.RuntimeMark{Schema: 1, ProcessID: "daemon-process", Component: "daemon", Scenario: "resize", Run: 7, Sequence: 9, RequestID: 11, Epoch: 13, Kind: kind, Valid: true}
	158	+	}
	159	+
	160	+	func readJSONLRecords(t *testing.T, path string) []map[string]any {
	161	+	t.Helper()
	162	+	f, err := os.Open(path)
	163	+	if err != nil {
	164	+	t.Fatalf("Open(%q): %v", path, err)
	165	+	}
	166	+	defer func() { _ = f.Close() }()
	167	+	var records []map[string]any
	168	+	scanner := bufio.NewScanner(f)
	169	+	for scanner.Scan() {
	170	+	var record map[string]any
	171	+	if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
	172	+	t.Fatalf("invalid JSONL record %q: %v", scanner.Text(), err)
	173	+	}
	174	+	records = append(records, record)
	175	+	}
	176	+	if err := scanner.Err(); err != nil {
	177	+	t.Fatalf("reading JSONL: %v", err)
	178	+	}
	179	+	return records
	180	+	}
	181	+
	182	+	type fixedRuntimeClock struct{ now time.Time }
	183	+
	184	+	func (c fixedRuntimeClock) Now() time.Time { return c.now }
	185	+	func (c fixedRuntimeClock) NewTimer(d time.Duration) ports.Timer {
	186	+	return fixedRuntimeTimer{timer: time.NewTimer(d)}
	187	+	}
	188	+
	189	+	type fixedRuntimeTimer struct{ timer *time.Timer }
	190	+
	191	+	func (t fixedRuntimeTimer) C() <-chan time.Time { return t.timer.C }
	192	+	func (t fixedRuntimeTimer) Reset(d time.Duration) bool { return t.timer.Reset(d) }
	193	+	func (t fixedRuntimeTimer) Stop() bool { return t.timer.Stop() }

M internal/adapters/remote/remote_factory.go +17 -8


@@ -11,35 +11,44 @@ import (
11	11		)
12	12
13	13		// DialerFactory selects the requested remote transport adapter.
14		-	type DialerFactory struct{}
	14	+	type DialerFactory struct{ observer ports.RuntimeObserver }
15	15
16	16		var _ ports.RemoteDialerFactory = DialerFactory{}
17	17
18	18		func NewDialerFactory() DialerFactory { return DialerFactory{} }
19	19
20		-	func (DialerFactory) DialerForRemote(target string, session string, mode ports.RemoteTransportMode, log *slog.Logger) (ports.Dialer, error) {
	20	+	// NewDialerFactoryWithRuntimeObserver wires the one process-local sink into
	21	+	// whichever concrete remote carriage adapter is selected.
	22	+	func NewDialerFactoryWithRuntimeObserver(observer ports.RuntimeObserver) DialerFactory {
	23	+	return DialerFactory{observer: observer}
	24	+	}
	25	+
	26	+	func (f DialerFactory) DialerForRemote(target string, session string, mode ports.RemoteTransportMode, log *slog.Logger) (ports.Dialer, error) {
21	27		switch mode {
22	28		case ports.RemoteTransportUDP:
23	29		if log != nil {
24	30		log.Info("remote transport selected", "mode", mode, "target", target, "session", session)
25	31		}
26		-	return dgram.NewRemoteDialerWithLogger(target, session, log), nil
	32	+	dialer := dgram.NewRemoteDialerWithLogger(target, session, log)
	33	+	dialer.RuntimeObserver = f.observer
	34	+	return dialer, nil
27	35		case ports.RemoteTransportStdio:
28	36		if log != nil {
29	37		log.Info("remote transport selected", "mode", mode, "target", target, "session", session)
30	38		}
31		-	return stdioDialer{target: target, session: session, log: log}, nil
	39	+	return stdioDialer{target: target, session: session, log: log, observer: f.observer}, nil
32	40		default:
33	41		return nil, fmt.Errorf("vev: unsupported remote transport %q", mode)
34	42		}
35	43		}
36	44
37	45		type stdioDialer struct {
38		-	target string
39		-	session string
40		-	log *slog.Logger
	46	+	target string
	47	+	session string
	48	+	log *slog.Logger
	49	+	observer ports.RuntimeObserver
41	50		}
42	51
43	52		func (d stdioDialer) Dial(ctx context.Context) (ports.Transport, error) {
44		-	return sshstdio.DialContext(ctx, d.target, d.session, d.log)
	53	+	return sshstdio.DialContextWithRuntimeObserver(ctx, d.target, d.session, d.log, sshstdio.WithRuntimeObserver(d.observer))
45	54		}

M internal/adapters/sshstdio/transport.go +127 -14


@@ -34,9 +34,23 @@ var (
34	34		type closeFunc func() error
35	35		type eofErrFunc func() error
36	36
	37	+	type Option func(*transport)
	38	+
	39	+	// WithRuntimeObserver enables process-local transport marks; it deliberately
	40	+	// takes only an observer so a carriage adapter never owns trace time.
	41	+	func WithRuntimeObserver(observer ports.RuntimeObserver) Option {
	42	+	return func(t *transport) { t.observer = observer }
	43	+	}
	44	+
37	45		// NewTransport wraps separate reader/writer streams as a framed Transport.
38		-	func NewTransport(r io.Reader, w io.Writer, closeFn closeFunc) ports.Transport {
39		-	return newTransport(r, w, closeFn, nil)
	46	+	func NewTransport(r io.Reader, w io.Writer, closeFn closeFunc, opts ...Option) ports.Transport {
	47	+	t := newTransport(r, w, closeFn, nil).(*transport)
	48	+	for _, opt := range opts {
	49	+	if opt != nil {
	50	+	opt(t)
	51	+	}
	52	+	}
	53	+	return t
40	54		}
41	55
42	56		func newTransport(r io.Reader, w io.Writer, closeFn closeFunc, eofErr eofErrFunc) ports.Transport {

@@ -47,18 +61,25 @@ func newTransport(r io.Reader, w io.Writer, closeFn closeFunc, eofErr eofErrFunc
47	61		}
48	62
49	63		type transport struct {
50		-	r io.Reader
51		-	w io.Writer
52		-	close closeFunc
53		-	eofErr eofErrFunc
	64	+	r io.Reader
	65	+	w io.Writer
	66	+	close closeFunc
	67	+	eofErr eofErrFunc
	68	+	observer ports.RuntimeObserver
	69	+	operationMu sync.Mutex
	70	+	operationCount int
	71	+	closing bool
	72	+	operationsDone chan struct{}
54	73
55	74		mu sync.Mutex
56	75		readBuf []byte
57	76		}
58	77
59	78		func (t *transport) Send(f ports.Frame) error {
	79	+	end := t.beginOperation(ports.RuntimeAdapterSendStart, uint64(len(f.Payload)))
60	80		n := 1 + len(f.Payload)
61	81		if n > maxFrameLen {
	82	+	end(false)
62	83		return ErrFrameTooLarge
63	84		}
64	85

@@ -68,22 +89,28 @@ func (t *transport) Send(f ports.Frame) error {
68	89		copy(buf[frameHeaderLen+1:], f.Payload)
69	90
70	91		t.mu.Lock()
71		-	defer t.mu.Unlock()
72	92		_, err := t.w.Write(buf)
	93	+	t.mu.Unlock()
	94	+	end(err == nil)
73	95		return err
74	96		}
75	97
76	98		func (t *transport) Recv() (ports.Frame, error) {
	99	+	end := t.beginOperation(ports.RuntimeAdapterReceiveStart, 0)
77	100		var hdr [frameHeaderLen]byte
78	101		if _, err := io.ReadFull(t.r, hdr[:]); err != nil {
79		-	return ports.Frame{}, t.mapEOFError(err)
	102	+	err = t.mapEOFError(err)
	103	+	end(false)
	104	+	return ports.Frame{}, err
80	105		}
81	106
82	107		n := binary.BigEndian.Uint32(hdr[:])
83	108		if n == 0 {
	109	+	end(false)
84	110		return ports.Frame{}, ErrZeroLengthFrame
85	111		}
86	112		if n > maxFrameLen {
	113	+	end(false)
87	114		return ports.Frame{}, ErrFrameTooLarge
88	115		}
89	116

@@ -93,13 +120,35 @@ func (t *transport) Recv() (ports.Frame, error) {
93	120		t.readBuf = t.readBuf[:n]
94	121		}
95	122		if _, err := io.ReadFull(t.r, t.readBuf); err != nil {
96		-	return ports.Frame{}, t.mapEOFError(err)
	123	+	err = t.mapEOFError(err)
	124	+	end(false)
	125	+	return ports.Frame{}, err
97	126		}
98	127
99	128		payload := append([]byte(nil), t.readBuf[1:]...)
	129	+	end(true)
100	130		return ports.Frame{Type: ports.MsgType(t.readBuf[0]), Payload: payload}, nil
101	131		}
102	132
	133	+	func (t *transport) beginOperation(start ports.RuntimeMarkKind, bytes uint64) func(bool) {
	134	+	if t.observer == nil {
	135	+	return func(bool) {}
	136	+	}
	137	+	if !t.beginObservedOperation() {
	138	+	return func(bool) {}
	139	+	}
	140	+	correlation := ports.NewRuntimeCorrelation()
	141	+	t.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("sshstdio", correlation, start, bytes, true))
	142	+	end := ports.RuntimeAdapterSendEnd
	143	+	if start == ports.RuntimeAdapterReceiveStart {
	144	+	end = ports.RuntimeAdapterReceiveEnd
	145	+	}
	146	+	return func(valid bool) {
	147	+	defer t.finishObservedOperation()
	148	+	t.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("sshstdio", correlation, end, bytes, valid))
	149	+	}
	150	+	}
	151	+
103	152		func (t *transport) mapEOFError(err error) error {
104	153		if t.eofErr == nil \|\| (!errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF)) {
105	154		return err

@@ -110,7 +159,56 @@ func (t *transport) mapEOFError(err error) error {
110	159		return err
111	160		}
112	161
113		-	func (t *transport) Close() error { return t.close() }
	162	+	func (t *transport) Close() error {
	163	+	done := t.beginShutdown()
	164	+	// Closing the reader first releases a Recv blocked in io.ReadFull even if
	165	+	// the subprocess shutdown callback reports an error. This lets Recv emit
	166	+	// its failed end mark before the process observer is closed.
	167	+	var readErr error
	168	+	if r, ok := t.r.(io.Closer); ok {
	169	+	readErr = r.Close()
	170	+	}
	171	+	closeErr := t.close()
	172	+	if done != nil {
	173	+	<-done
	174	+	}
	175	+	if closeErr != nil {
	176	+	return closeErr
	177	+	}
	178	+	return readErr
	179	+	}
	180	+
	181	+	func (t *transport) beginObservedOperation() bool {
	182	+	t.operationMu.Lock()
	183	+	defer t.operationMu.Unlock()
	184	+	if t.closing {
	185	+	return false
	186	+	}
	187	+	if t.operationCount == 0 {
	188	+	t.operationsDone = make(chan struct{})
	189	+	}
	190	+	t.operationCount++
	191	+	return true
	192	+	}
	193	+
	194	+	func (t *transport) finishObservedOperation() {
	195	+	t.operationMu.Lock()
	196	+	defer t.operationMu.Unlock()
	197	+	t.operationCount--
	198	+	if t.operationCount == 0 {
	199	+	close(t.operationsDone)
	200	+	}
	201	+	}
	202	+
	203	+	func (t *transport) beginShutdown() <-chan struct{} {
	204	+	t.operationMu.Lock()
	205	+	defer t.operationMu.Unlock()
	206	+	t.closing = true
	207	+	if t.operationCount == 0 {
	208	+	return nil
	209	+	}
	210	+	return t.operationsDone
	211	+	}
114	212
115	213		type processWaiter struct {
116	214		cmd *exec.Cmd

@@ -222,18 +320,27 @@ func Dial(target, session string) (ports.Transport, error) {
222	320		return DialContext(context.Background(), target, session)
223	321		}
224	322
	323	+	// DialContextWithRuntimeObserver is the option-bearing production constructor.
	324	+	func DialContextWithRuntimeObserver(ctx context.Context, target, session string, logger *slog.Logger, opts ...Option) (ports.Transport, error) {
	325	+	return dialContext(ctx, target, session, logger, opts...)
	326	+	}
	327	+
225	328		// DialContext is like Dial, but the context is propagated to ssh startup so a
226	329		// canceled attach attempt interrupts the local ssh process. Callers may pass a
227	330		// logger to record ssh start failures and non-clean exits without logging the
228	331		// generated command line.
229	332		func DialContext(ctx context.Context, target, session string, logger ...*slog.Logger) (ports.Transport, error) {
230		-	if err := ctx.Err(); err != nil {
231		-	return nil, err
232		-	}
233	333		var log *slog.Logger
234	334		if len(logger) > 0 {
235	335		log = logger[0]
236	336		}
	337	+	return dialContext(ctx, target, session, log)
	338	+	}
	339	+
	340	+	func dialContext(ctx context.Context, target, session string, log *slog.Logger, opts ...Option) (ports.Transport, error) {
	341	+	if err := ctx.Err(); err != nil {
	342	+	return nil, err
	343	+	}
237	344		spec := BuildCommand(target, session)
238	345		cmd := exec.CommandContext(ctx, spec.Path, spec.Args...)
239	346		stdin, err := cmd.StdinPipe()

@@ -254,5 +361,11 @@ func DialContext(ctx context.Context, target, session string, logger ...*slog.Lo
254	361		}
255	362
256	363		waiter := newProcessWaiter(cmd, stdin, &stderr, sshCloseTimeout, log, target, session)
257		-	return newTransport(stdout, stdin, waiter.close, waiter.eofErr), nil
	364	+	transport := newTransport(stdout, stdin, waiter.close, waiter.eofErr).(*transport)
	365	+	for _, opt := range opts {
	366	+	if opt != nil {
	367	+	opt(transport)
	368	+	}
	369	+	}
	370	+	return transport, nil
258	371		}

A internal/adapters/sshstdio/transport_observability_test.go +173 -0

@@ -0,0 +1,173 @@
	1	+	package sshstdio
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"os"
	8	+	"strings"
	9	+	"sync"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"github.com/bnema/vev/internal/adapters/observability"
	14	+	"github.com/bnema/vev/internal/ports"
	15	+	)
	16	+
	17	+	func TestTransportObservabilitySSHStdioEOFEndsReceive(t *testing.T) {
	18	+	tracePath := t.TempDir() + "/ssh-eof.jsonl"
	19	+	observer, closer, err := observability.NewJSONL(tracePath, sshRuntimeClock{now: time.Unix(0, 309)}, "ssh-process")
	20	+	if err != nil {
	21	+	t.Fatalf("NewJSONL() error = %v", err)
	22	+	}
	23	+	if _, err := NewTransport(bytes.NewReader(nil), &bytes.Buffer{}, nil, WithRuntimeObserver(observer)).Recv(); err == nil {
	24	+	t.Fatal("Recv() error = nil at EOF")
	25	+	}
	26	+	if err := closer.Close(); err != nil {
	27	+	t.Fatalf("trace Close() error = %v", err)
	28	+	}
	29	+	assertSSHReceiveFailurePair(t, tracePath)
	30	+	}
	31	+
	32	+	func TestTransportObservabilitySSHStdioCloseEndsBlockedReceive(t *testing.T) {
	33	+	tracePath := t.TempDir() + "/ssh-close.jsonl"
	34	+	observer, closer, err := observability.NewJSONL(tracePath, sshRuntimeClock{now: time.Unix(0, 311)}, "ssh-process")
	35	+	if err != nil {
	36	+	t.Fatalf("NewJSONL() error = %v", err)
	37	+	}
	38	+	reader := &sshShutdownReader{entered: make(chan struct{}), done: make(chan struct{})}
	39	+	shutdownErr := errors.New("ssh shutdown failed")
	40	+	transport := NewTransport(reader, &bytes.Buffer{}, func() error { return shutdownErr }, WithRuntimeObserver(observer))
	41	+	recvDone := make(chan error, 1)
	42	+	go func() { _, err := transport.Recv(); recvDone <- err }()
	43	+	<-reader.entered
	44	+	if err := transport.Close(); !errors.Is(err, shutdownErr) {
	45	+	t.Fatalf("Close() error = %v, want %v", err, shutdownErr)
	46	+	}
	47	+	if err := <-recvDone; err == nil {
	48	+	t.Fatal("Recv() error = nil after Close()")
	49	+	}
	50	+	if err := closer.Close(); err != nil {
	51	+	t.Fatalf("trace Close() error = %v", err)
	52	+	}
	53	+	assertSSHReceiveFailurePair(t, tracePath)
	54	+	}
	55	+
	56	+	type sshShutdownReader struct {
	57	+	entered chan struct{}
	58	+	done chan struct{}
	59	+	once sync.Once
	60	+	}
	61	+
	62	+	func (r *sshShutdownReader) Read([]byte) (int, error) {
	63	+	r.once.Do(func() { close(r.entered) })
	64	+	<-r.done
	65	+	return 0, errors.New("reader closed")
	66	+	}
	67	+
	68	+	func (r *sshShutdownReader) Close() error {
	69	+	r.once.Do(func() { close(r.entered) })
	70	+	close(r.done)
	71	+	return nil
	72	+	}
	73	+
	74	+	func assertSSHReceiveFailurePair(t *testing.T, path string) {
	75	+	t.Helper()
	76	+	data, err := os.ReadFile(path)
	77	+	if err != nil {
	78	+	t.Fatalf("ReadFile(trace): %v", err)
	79	+	}
	80	+	var starts, ends []struct {
	81	+	ProcessID string `json:"process_id"`
	82	+	Scenario string `json:"scenario"`
	83	+	Run uint64 `json:"run"`
	84	+	Sequence uint64 `json:"sequence"`
	85	+	RequestID uint64 `json:"request_id"`
	86	+	Epoch uint64 `json:"epoch"`
	87	+	Kind string `json:"kind"`
	88	+	Valid bool `json:"valid"`
	89	+	}
	90	+	for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
	91	+	var mark struct {
	92	+	ProcessID string `json:"process_id"`
	93	+	Scenario string `json:"scenario"`
	94	+	Run uint64 `json:"run"`
	95	+	Sequence uint64 `json:"sequence"`
	96	+	RequestID uint64 `json:"request_id"`
	97	+	Epoch uint64 `json:"epoch"`
	98	+	Kind string `json:"kind"`
	99	+	Valid bool `json:"valid"`
	100	+	}
	101	+	if err := json.Unmarshal([]byte(line), &mark); err != nil {
	102	+	t.Fatalf("Unmarshal(trace): %v", err)
	103	+	}
	104	+	switch mark.Kind {
	105	+	case "adapter_receive_start":
	106	+	starts = append(starts, mark)
	107	+	case "adapter_receive_end":
	108	+	ends = append(ends, mark)
	109	+	}
	110	+	}
	111	+	if len(starts) != 1 \|\| len(ends) != 1 {
	112	+	t.Fatalf("receive spans start=%d end=%d, want exactly one pair", len(starts), len(ends))
	113	+	}
	114	+	start, end := starts[0], ends[0]
	115	+	if start.ProcessID != end.ProcessID \|\| start.Scenario != end.Scenario \|\| start.Run != end.Run \|\| start.Sequence != end.Sequence \|\| start.RequestID != end.RequestID \|\| start.Epoch != end.Epoch {
	116	+	t.Fatalf("mismatched receive correlation: start=%+v end=%+v", start, end)
	117	+	}
	118	+	if !start.Valid \|\| end.Valid {
	119	+	t.Fatalf("receive validity start=%t end=%t, want true/false", start.Valid, end.Valid)
	120	+	}
	121	+	}
	122	+
	123	+	func TestTransportObservabilitySSHStdioPreservesCarriage(t *testing.T) {
	124	+	tracePath := t.TempDir() + "/ssh.jsonl"
	125	+	observer, closer, err := observability.NewJSONL(tracePath, sshRuntimeClock{now: time.Unix(0, 307)}, "ssh-process")
	126	+	if err != nil {
	127	+	t.Fatalf("NewJSONL() error = %v", err)
	128	+	}
	129	+
	130	+	frame := ports.Frame{Type: ports.MsgOutput, Payload: []byte("stdio bytes stay exact")}
	131	+	var observed, baseline bytes.Buffer
	132	+	if err := NewTransport(bytes.NewReader(nil), &observed, nil, WithRuntimeObserver(observer)).Send(frame); err != nil {
	133	+	t.Fatalf("observed Send() error = %v", err)
	134	+	}
	135	+	if err := NewTransport(bytes.NewReader(nil), &baseline, nil).Send(frame); err != nil {
	136	+	t.Fatalf("baseline Send() error = %v", err)
	137	+	}
	138	+	if !bytes.Equal(observed.Bytes(), baseline.Bytes()) {
	139	+	t.Fatalf("observer changed SSH stdio wire bytes: got %x, want %x", observed.Bytes(), baseline.Bytes())
	140	+	}
	141	+	received, err := NewTransport(bytes.NewReader(baseline.Bytes()), &bytes.Buffer{}, nil, WithRuntimeObserver(observer)).Recv()
	142	+	if err != nil {
	143	+	t.Fatalf("observed Recv() error = %v", err)
	144	+	}
	145	+	if received.Type != frame.Type \|\| !bytes.Equal(received.Payload, frame.Payload) {
	146	+	t.Fatalf("observed Recv() = %#v, want %#v", received, frame)
	147	+	}
	148	+	if err := closer.Close(); err != nil {
	149	+	t.Fatalf("trace Close() error = %v", err)
	150	+	}
	151	+	trace, err := os.ReadFile(tracePath)
	152	+	if err != nil {
	153	+	t.Fatalf("ReadFile(trace): %v", err)
	154	+	}
	155	+	for _, kind := range []string{"adapter_send_start", "adapter_send_end", "adapter_receive_start", "adapter_receive_end"} {
	156	+	if !strings.Contains(string(trace), `"kind":"`+kind+`"`) {
	157	+	t.Errorf("trace missing %s", kind)
	158	+	}
	159	+	}
	160	+	}
	161	+
	162	+	type sshRuntimeClock struct{ now time.Time }
	163	+
	164	+	func (c sshRuntimeClock) Now() time.Time { return c.now }
	165	+	func (c sshRuntimeClock) NewTimer(d time.Duration) ports.Timer {
	166	+	return sshRuntimeTimer{timer: time.NewTimer(d)}
	167	+	}
	168	+
	169	+	type sshRuntimeTimer struct{ timer *time.Timer }
	170	+
	171	+	func (t sshRuntimeTimer) C() <-chan time.Time { return t.timer.C }
	172	+	func (t sshRuntimeTimer) Reset(d time.Duration) bool { return t.timer.Reset(d) }
	173	+	func (t sshRuntimeTimer) Stop() bool { return t.timer.Stop() }

M internal/app/daemon_integration_test.go +5 -5


@@ -74,7 +74,7 @@ func recvPump(tr ports.Transport) *pump {
74	74		// attach dials, handshakes, and returns the transport plus its frame pump.
75	75		func attach(t testing.T, dir string, intent uint8, name string, sz domain.Size) (ports.Transport, pump) {
76	76		t.Helper()
77		-	tr, err := ipc.Dial(dir)
	77	+	tr, err := ipc.DialContext(context.Background(), dir)
78	78		require.NoError(t, err)
79	79		hello := ports.Hello{Version: ports.ProtocolVersion, Intent: intent, Name: name, Size: sz, TermEnv: "xterm-256color"}
80	80		require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgHello, Payload: ports.MarshalHello(hello)}))

@@ -91,7 +91,7 @@ func attach(t *testing.T, dir string, intent uint8, name string, sz domain.Size)
91	91
92	92		func listRemoteSessions(t *testing.T, dir string) ports.Sessions {
93	93		t.Helper()
94		-	tr, err := ipc.Dial(dir)
	94	+	tr, err := ipc.DialContext(context.Background(), dir)
95	95		require.NoError(t, err)
96	96		defer func() { _ = tr.Close() }()
97	97		require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgList, Payload: ports.MarshalList(ports.List{})}))

@@ -104,7 +104,7 @@ func listRemoteSessions(t *testing.T, dir string) ports.Sessions {
104	104		}
105	105
106	106		func killAll(dir string) error {
107		-	tr, err := ipc.Dial(dir)
	107	+	tr, err := ipc.DialContext(context.Background(), dir)
108	108		if err != nil {
109	109		return err
110	110		}

@@ -248,7 +248,7 @@ func TestIntegration_CommandPaletteRenamesEphemeralSession(t *testing.T) {
248	248		text = awaitScreenText(t, p, sz, " work ")
249	249		require.NotContains(t, text, "work*")
250	250
251		-	listTr, err := ipc.Dial(dir)
	251	+	listTr, err := ipc.DialContext(context.Background(), dir)
252	252		require.NoError(t, err)
253	253		defer func() { _ = listTr.Close() }()
254	254		require.NoError(t, listTr.Send(ports.Frame{Type: ports.MsgList, Payload: ports.MarshalList(ports.List{})}))

@@ -367,7 +367,7 @@ func TestIntegration_KillAllShutsDownDaemon(t *testing.T) {
367	367		tr2, _ := attach(t, dir, ports.IntentNew, "two", sz)
368	368		require.NoError(t, tr2.Close())
369	369
370		-	killTr, err := ipc.Dial(dir)
	370	+	killTr, err := ipc.DialContext(context.Background(), dir)
371	371		require.NoError(t, err)
372	372		defer func() { _ = killTr.Close() }()
373	373		require.NoError(t, killTr.Send(ports.Frame{Type: ports.MsgKill, Payload: ports.MarshalKill(ports.Kill{All: true})}))

M internal/app/run.go +137 -19


@@ -22,6 +22,7 @@ import (
22	22		"path/filepath"
23	23		"strconv"
24	24		"strings"
	25	+	"sync"
25	26		"syscall"
26	27		"text/tabwriter"
27	28		"time"

@@ -31,6 +32,7 @@ import (
31	32		"github.com/bnema/vev/internal/adapters/config"
32	33		"github.com/bnema/vev/internal/adapters/dgram"
33	34		"github.com/bnema/vev/internal/adapters/ipc"
	35	+	"github.com/bnema/vev/internal/adapters/observability"
34	36		"github.com/bnema/vev/internal/adapters/pty"
35	37		remoteadapter "github.com/bnema/vev/internal/adapters/remote"
36	38		"github.com/bnema/vev/internal/adapters/shellcmd"

@@ -238,6 +240,72 @@ func dispatch(ctx context.Context, cmd command) error {
238	240		}
239	241		}
240	242
	243	+	// performanceTrace creates one serialized timestamp owner for this process.
	244	+	// An empty trace environment leaves all production behavior and wire bytes
	245	+	// unchanged.
	246	+	func performanceTrace(clk ports.Clock) (ports.RuntimeObserver, io.Closer, error) {
	247	+	path, processID := os.Getenv("VEV_PERF_TRACE"), os.Getenv("VEV_PERF_PROCESS_ID")
	248	+	if path == "" {
	249	+	return nil, nil, nil
	250	+	}
	251	+	observer, closer, err := observability.NewJSONL(path, clk, processID)
	252	+	if err != nil {
	253	+	return nil, nil, err
	254	+	}
	255	+	// The harness supplies these manifest fields to every launched role. Keep a
	256	+	// valid standalone trace for operators that set only the original trace
	257	+	// variables, while ensuring harness traces match their process mapping.
	258	+	inputs := ports.RuntimeCorrelationInputs{Scenario: os.Getenv("VEV_PERF_SCENARIO"), Run: 1}
	259	+	if inputs.Scenario == "" {
	260	+	inputs.Scenario = "runtime"
	261	+	}
	262	+	if rawRun := os.Getenv("VEV_PERF_RUN"); rawRun != "" {
	263	+	inputs.Run, err = strconv.ParseUint(rawRun, 10, 64)
	264	+	if err != nil \|\| inputs.Run == 0 {
	265	+	_ = closer.Close()
	266	+	return nil, nil, fmt.Errorf("invalid VEV_PERF_RUN %q", rawRun)
	267	+	}
	268	+	}
	269	+	observer, err = ports.NewRuntimeCorrelationObserver(observer, inputs)
	270	+	if err != nil {
	271	+	_ = closer.Close()
	272	+	return nil, nil, err
	273	+	}
	274	+	reporter := ports.NewSerializedRuntimeObserver(observer, runtimeTraceQueueDepth)
	275	+	return reporter, &runtimeTraceCloser{reporter: reporter, closer: closer}, nil
	276	+	}
	277	+
	278	+	// runtimeTraceQueueDepth bounds all process-local trace producer handoffs.
	279	+	// A full queue emits the serialized diagnostic gap rather than delaying a
	280	+	// terminal, transport, or ACK progress path.
	281	+	const runtimeTraceQueueDepth = 256
	282	+
	283	+	// runtimeTraceCloser owns the two-stage trace shutdown: drain the reporter
	284	+	// before closing the concrete timestamp/file owner. sync.Once makes every
	285	+	// deferred and explicit close path safe without closing shared workers twice.
	286	+	type runtimeTraceCloser struct {
	287	+	reporter ports.SerializedRuntimeObserver
	288	+	closer io.Closer
	289	+	once sync.Once
	290	+	err error
	291	+	}
	292	+
	293	+	func (c *runtimeTraceCloser) Close() error {
	294	+	if c == nil {
	295	+	return nil
	296	+	}
	297	+	c.once.Do(func() {
	298	+	if c.reporter != nil {
	299	+	c.reporter.Flush()
	300	+	c.reporter.Close()
	301	+	}
	302	+	if c.closer != nil {
	303	+	c.err = c.closer.Close()
	304	+	}
	305	+	})
	306	+	return c.err
	307	+	}
	308	+
241	309		func configureLogging(component logging.Component, rotateAtRuntime bool) (*slog.Logger, io.Closer, error) {
242	310		return logging.Setup(logging.Config{
243	311		Dir: platform.StateDir(),

@@ -285,7 +353,16 @@ func runDaemon() error {
285	353		}
286	354		defer func() { _ = logCloser.Close() }()
287	355
288		-	ln, err := ipc.Listen(ipc.SocketDir())
	356	+	clk := clock.New()
	357	+	observer, observerCloser, err := performanceTrace(clk)
	358	+	if err != nil {
	359	+	return fmt.Errorf("vev: performance trace: %w", err)
	360	+	}
	361	+	if observerCloser != nil {
	362	+	defer func() { _ = observerCloser.Close() }()
	363	+	}
	364	+	// Every accepted local connection shares this process's serialized sink.
	365	+	ln, err := ipc.Listen(ipc.SocketDir(), ipc.WithRuntimeObserver(observer))
289	366		if err != nil {
290	367		log.Error("daemon listen failed", "socket_dir", ipc.SocketDir(), "err", err)
291	368		return fmt.Errorf("vev: daemon listen: %w", err)

@@ -309,6 +386,9 @@ func runDaemon() error {
309	386
310	387		log.Info("daemon starting", "socket", ln.Addr())
311	388		daemonOpts := []daemon.Option(nil)
	389	+	if observer != nil {
	390	+	daemonOpts = append(daemonOpts, daemon.WithRuntimeObserver(observer))
	391	+	}
312	392		configPath := platform.ConfigPath()
313	393		cfg, warnings, err := config.Load(configPath)
314	394		if err != nil {

@@ -327,7 +407,6 @@ func runDaemon() error {
327	407		log.Info("session persistence enabled", "path", storePath)
328	408		daemonOpts = append(daemonOpts, daemon.WithStore(store))
329	409		}
330		-	clk := clock.New()
331	410		d := daemon.New(pty.NewFactory(), clk, log, daemonOpts...)
332	411		watchCtx, stopWatch := context.WithCancel(ctx)
333	412		defer stopWatch()

@@ -357,13 +436,25 @@ func runAttach(ctx context.Context, intent uint8, name, remoteTarget string) err
357	436		}
358	437		defer func() { _ = logCloser.Close() }()
359	438
	439	+	clk := clock.New()
	440	+	observer, observerCloser, err := performanceTrace(clk)
	441	+	if err != nil {
	442	+	return fmt.Errorf("vev: performance trace: %w", err)
	443	+	}
	444	+	if observerCloser != nil {
	445	+	defer func() { _ = observerCloser.Close() }()
	446	+	}
360	447		return runAttachWithDeps(ctx, intent, name, remoteTarget, os.Getenv("VEV"), log, runAttachDeps{
361		-	localDialer: defaultLocalDialer,
362		-	remoteDialerFactory: defaultRemoteDialerFactory(),
	448	+	localDialer: func() ports.Dialer {
	449	+	return localDaemonDialer{dir: ipc.SocketDir(), observer: observer}
	450	+	},
	451	+	remoteDialerFactory: remoteadapter.NewDialerFactoryWithRuntimeObserver(observer),
363	452		selectedRemoteTransport: os.Getenv(envRemoteTransport),
364		-	runClient: client.Run,
365		-	createDetached: createDetachedLocalSession,
366		-	clipboard: clipboard.New(),
	453	+	runClient: func(ctx context.Context, dialer ports.Dialer, terminal ports.Terminal, behaviorClock ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger) error {
	454	+	return client.RunWithRuntimeObserver(ctx, dialer, terminal, behaviorClock, intent, name, remote, clipboard, log, observer)
	455	+	},
	456	+	createDetached: createDetachedLocalSession,
	457	+	clipboard: clipboard.New(),
367	458		})
368	459		}
369	460

@@ -499,10 +590,19 @@ func isDaemonVersionDrift(err error) bool {
499	590		return protocolErr.Code == ports.ErrVersionMismatch \|\| protocolErr.Text == legacyMalformedHelloSignal
500	591		}
501	592
502		-	type localDaemonDialer struct{ dir string }
	593	+	type localDaemonDialer struct {
	594	+	dir string
	595	+	observer ports.RuntimeObserver
	596	+	}
503	597
504	598		func (d localDaemonDialer) Dial(ctx context.Context) (ports.Transport, error) {
505		-	return ensureDaemon(ctx, d.dir, realDial, realSpawn, defaultBackoff)
	599	+	dial := realDial
	600	+	if d.observer != nil {
	601	+	dial = func(ctx context.Context, dir string) (ports.Transport, error) {
	602	+	return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(d.observer))
	603	+	}
	604	+	}
	605	+	return ensureDaemon(ctx, d.dir, dial, realSpawn, defaultBackoff)
506	606		}
507	607
508	608		func detachedLocalHello(name, cwd string) ports.Hello {

@@ -584,7 +684,7 @@ func requestDaemonStop(ctx context.Context) error {
584	684
585	685		done := make(chan error, 1)
586	686		go func() {
587		-	transport, err := realDial(ipc.SocketDir())
	687	+	transport, err := realDial(ctx, ipc.SocketDir())
588	688		if err != nil {
589	689		done <- fmt.Errorf("vev: no daemon running")
590	690		return

@@ -629,13 +729,22 @@ func runStdio(ctx context.Context) error {
629	729		defer func() { _ = logCloser.Close() }()
630	730		log.Debug("stdio proxy starting")
631	731
632		-	transport, err := ensureDaemon(ctx, ipc.SocketDir(), realDial, realSpawn, defaultBackoff)
	732	+	clk := clock.New()
	733	+	observer, observerCloser, err := performanceTrace(clk)
	734	+	if err != nil {
	735	+	return fmt.Errorf("vev: performance trace: %w", err)
	736	+	}
	737	+	if observerCloser != nil {
	738	+	defer func() { _ = observerCloser.Close() }()
	739	+	}
	740	+	transport, err := ensureDaemon(ctx, ipc.SocketDir(), func(ctx context.Context, dir string) (ports.Transport, error) {
	741	+	return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(observer))
	742	+	}, realSpawn, defaultBackoff)
633	743		if err != nil {
634	744		return err
635	745		}
636	746		defer func() { _ = transport.Close() }()
637		-
638		-	stdio := sshstdio.NewTransport(os.Stdin, os.Stdout, nil)
	747	+	stdio := sshstdio.NewTransport(os.Stdin, os.Stdout, nil, sshstdio.WithRuntimeObserver(observer))
639	748		return proxyTransports(ctx, stdio, transport, log)
640	749		}
641	750

@@ -727,6 +836,13 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) error {
727	836		defer func() { _ = logCloser.Close() }()
728	837		log.Debug("udp proxy starting", "session", session)
729	838
	839	+	observer, observerCloser, err := performanceTrace(clock.New())
	840	+	if err != nil {
	841	+	return fmt.Errorf("vev: performance trace: %w", err)
	842	+	}
	843	+	if observerCloser != nil {
	844	+	defer func() { _ = observerCloser.Close() }()
	845	+	}
730	846		portRange, err := parseUDPPortRange(os.Getenv(envUDPPortRange))
731	847		if err != nil {
732	848		return err

@@ -740,14 +856,16 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) error {
740	856		if _, err := rand.Read(key); err != nil {
741	857		return err
742	858		}
743		-	daemonTr, err := ensureDaemon(ctx, ipc.SocketDir(), realDial, realSpawn, defaultBackoff)
	859	+	daemonTr, err := ensureDaemon(ctx, ipc.SocketDir(), func(ctx context.Context, dir string) (ports.Transport, error) {
	860	+	return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(observer))
	861	+	}, realSpawn, defaultBackoff)
744	862		if err != nil {
745	863		return err
746	864		}
747	865		defer func() { _ = daemonTr.Close() }()
748	866		udpOptions := udpProxyClientTransportOptions
749	867		udpOptions.Observe = dgram.DiagnosticLogObserver(log)
750		-	dg, err := dgram.NewTransportWithOptions(conn, nil, key, 2, 1, udpOptions)
	868	+	dg, err := dgram.NewTransportWithOptions(conn, nil, key, 2, 1, udpOptions, dgram.WithRuntimeObserver(observer))
751	869		if err != nil {
752	870		return err
753	871		}

@@ -890,8 +1008,8 @@ func proxyTransports(ctx context.Context, a, b ports.Transport, log *slog.Logger
890	1008
891	1009		// runList prints the daemon's session listing. With no daemon running, it
892	1010		// falls back to the persisted stopped-session records.
893		-	func runList(_ context.Context) error {
894		-	transport, err := realDial(ipc.SocketDir())
	1011	+	func runList(ctx context.Context) error {
	1012	+	transport, err := realDial(ctx, ipc.SocketDir())
895	1013		if err != nil {
896	1014		records, loadErr := persist.LoadReadOnly(platform.StateDir())
897	1015		if loadErr != nil {

@@ -955,8 +1073,8 @@ func printSessions(w io.Writer, sessions []ports.SessionInfo) {
955	1073		}
956	1074
957	1075		// runKill asks the daemon to terminate a named session, every session, or the daemon.
958		-	func runKill(_ context.Context, name string, all, daemon bool) error {
959		-	transport, err := realDial(ipc.SocketDir())
	1076	+	func runKill(ctx context.Context, name string, all, daemon bool) error {
	1077	+	transport, err := realDial(ctx, ipc.SocketDir())
960	1078		if err != nil {
961	1079		if name != "" && !all && !daemon {
962	1080		storePath := persist.StorePath(platform.StateDir())

M internal/app/spawn.go +14 -5


@@ -32,7 +32,7 @@ const staleLockAge = 10 * time.Second
32	32
33	33		// dialFunc dials the daemon socket in dir. Injected so tests can drive the
34	34		// spawn/backoff logic without a real socket.
35		-	type dialFunc func(dir string) (ports.Transport, error)
	35	+	type dialFunc func(ctx context.Context, dir string) (ports.Transport, error)
36	36
37	37		// spawnFunc launches a detached daemon process. Injected so tests never
38	38		// re-exec a real binary.

@@ -58,7 +58,10 @@ var defaultBackoff = backoffConfig{
58	58		// spawner via an mkdir lock (losers simply wait), then retry-dials with
59	59		// backoff until the socket is live or the budget expires.
60	60		func ensureDaemon(ctx context.Context, dir string, dial dialFunc, spawn spawnFunc, cfg backoffConfig) (ports.Transport, error) {
61		-	if t, err := dial(dir); err == nil {
	61	+	if err := ctx.Err(); err != nil {
	62	+	return nil, err
	63	+	}
	64	+	if t, err := dial(ctx, dir); err == nil {
62	65		slog.Debug("daemon already reachable", "socket_dir", dir)
63	66		return t, nil
64	67		}

@@ -73,6 +76,9 @@ func ensureDaemon(ctx context.Context, dir string, dial dialFunc, spawn spawnFun
73	76		// rather than spawning a second daemon.
74	77		defer release()
75	78		slog.Info("spawning daemon", "socket_dir", dir)
	79	+	if err := ctx.Err(); err != nil {
	80	+	return nil, err
	81	+	}
76	82		if err := spawn(); err != nil {
77	83		slog.Error("daemon spawn failed", "err", err)
78	84		return nil, fmt.Errorf("vev: spawning daemon: %w", err)

@@ -120,7 +126,10 @@ func retryDial(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig
120	126		deadline := time.Now().Add(cfg.total)
121	127		backoff := cfg.initial
122	128		for {
123		-	if t, err := dial(dir); err == nil {
	129	+	if err := ctx.Err(); err != nil {
	130	+	return nil, err
	131	+	}
	132	+	if t, err := dial(ctx, dir); err == nil {
124	133		return t, nil
125	134		}
126	135		if !time.Now().Before(deadline) {

@@ -139,8 +148,8 @@ func retryDial(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig
139	148		}
140	149
141	150		// realDial is the production dialer.
142		-	func realDial(dir string) (ports.Transport, error) {
143		-	return ipc.Dial(dir)
	151	+	func realDial(ctx context.Context, dir string) (ports.Transport, error) {
	152	+	return ipc.DialContext(ctx, dir)
144	153		}
145	154
146	155		// realSpawn re-execs this binary as a detached daemon: a new session

M internal/app/spawn_test.go +53 -6


@@ -91,7 +91,7 @@ func TestAcquireSpawnLockTakesOverStaleLock(t *testing.T) {
91	91		func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {
92	92		want := portsmocks.NewMockTransport(t)
93	93		var calls atomic.Int32
94		-	dial := func(string) (ports.Transport, error) {
	94	+	dial := func(context.Context, string) (ports.Transport, error) {
95	95		if calls.Add(1) <= 3 {
96	96		return nil, errDialFailed
97	97		}

@@ -111,7 +111,7 @@ func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {
111	111		}
112	112
113	113		func TestRetryDialPermanentFailure(t *testing.T) {
114		-	dial := func(string) (ports.Transport, error) { return nil, errDialFailed }
	114	+	dial := func(context.Context, string) (ports.Transport, error) { return nil, errDialFailed }
115	115		_, err := retryDial(context.Background(), t.TempDir(), dial, fastBackoff)
116	116		if !errors.Is(err, ErrDaemonUnreachable) {
117	117		t.Fatalf("retryDial error = %v, want ErrDaemonUnreachable", err)

@@ -121,18 +121,65 @@ func TestRetryDialPermanentFailure(t *testing.T) {
121	121		func TestRetryDialContextCancelled(t *testing.T) {
122	122		ctx, cancel := context.WithCancel(context.Background())
123	123		cancel()
124		-	dial := func(string) (ports.Transport, error) { return nil, errDialFailed }
	124	+	var calls atomic.Int32
	125	+	dial := func(context.Context, string) (ports.Transport, error) {
	126	+	calls.Add(1)
	127	+	return nil, errDialFailed
	128	+	}
125	129		_, err := retryDial(ctx, t.TempDir(), dial, fastBackoff)
126	130		if !errors.Is(err, context.Canceled) {
127	131		t.Fatalf("retryDial error = %v, want context.Canceled", err)
128	132		}
	133	+	if calls.Load() != 0 {
	134	+	t.Fatalf("dial called %d times for canceled context, want 0", calls.Load())
	135	+	}
	136	+	}
	137	+
	138	+	func TestEnsureDaemonContextCancelledDoesNotDialOrSpawn(t *testing.T) {
	139	+	ctx, cancel := context.WithCancel(context.Background())
	140	+	cancel()
	141	+	dial := func(context.Context, string) (ports.Transport, error) {
	142	+	t.Fatal("ensureDaemon dialed with a canceled context")
	143	+	return nil, nil
	144	+	}
	145	+	spawn := func() error {
	146	+	t.Fatal("ensureDaemon spawned with a canceled context")
	147	+	return nil
	148	+	}
	149	+
	150	+	_, err := ensureDaemon(ctx, t.TempDir(), dial, spawn, fastBackoff)
	151	+	if !errors.Is(err, context.Canceled) {
	152	+	t.Fatalf("ensureDaemon error = %v, want context.Canceled", err)
	153	+	}
	154	+	}
	155	+
	156	+	func TestEnsureDaemonDoesNotSpawnWhenContextCancelsAfterFailedDial(t *testing.T) {
	157	+	ctx, cancel := context.WithCancel(context.Background())
	158	+	defer cancel()
	159	+	var spawnCalls atomic.Int32
	160	+	dial := func(context.Context, string) (ports.Transport, error) {
	161	+	cancel()
	162	+	return nil, errDialFailed
	163	+	}
	164	+	spawn := func() error {
	165	+	spawnCalls.Add(1)
	166	+	return nil
	167	+	}
	168	+
	169	+	_, err := ensureDaemon(ctx, filepath.Join(t.TempDir(), "vev"), dial, spawn, fastBackoff)
	170	+	if !errors.Is(err, context.Canceled) {
	171	+	t.Fatalf("ensureDaemon error = %v, want context.Canceled", err)
	172	+	}
	173	+	if got := spawnCalls.Load(); got != 0 {
	174	+	t.Fatalf("spawn called %d times after canceled failed dial, want 0", got)
	175	+	}
129	176		}
130	177
131	178		func TestEnsureDaemonSpawnsThenDials(t *testing.T) {
132	179		dir := filepath.Join(t.TempDir(), "vev")
133	180		want := portsmocks.NewMockTransport(t)
134	181		var dialCalls, spawnCalls atomic.Int32
135		-	dial := func(string) (ports.Transport, error) {
	182	+	dial := func(context.Context, string) (ports.Transport, error) {
136	183		// First (pre-spawn) dial fails; the post-spawn retry succeeds.
137	184		if dialCalls.Add(1) == 1 {
138	185		return nil, errDialFailed

@@ -163,7 +210,7 @@ func TestEnsureDaemonSpawnsThenDials(t *testing.T) {
163	210		func TestEnsureDaemonReturnsExistingDaemon(t *testing.T) {
164	211		want := portsmocks.NewMockTransport(t)
165	212		var spawnCalls atomic.Int32
166		-	dial := func(string) (ports.Transport, error) { return want, nil }
	213	+	dial := func(context.Context, string) (ports.Transport, error) { return want, nil }
167	214		spawn := func() error { spawnCalls.Add(1); return nil }
168	215
169	216		got, err := ensureDaemon(context.Background(), t.TempDir(), dial, spawn, fastBackoff)

@@ -180,7 +227,7 @@ func TestEnsureDaemonReturnsExistingDaemon(t *testing.T) {
180	227
181	228		func TestEnsureDaemonSpawnFailure(t *testing.T) {
182	229		spawnErr := errors.New("exec boom")
183		-	dial := func(string) (ports.Transport, error) { return nil, errDialFailed }
	230	+	dial := func(context.Context, string) (ports.Transport, error) { return nil, errDialFailed }
184	231		spawn := func() error { return spawnErr }
185	232
186	233		_, err := ensureDaemon(context.Background(), filepath.Join(t.TempDir(), "vev"), dial, spawn, fastBackoff)

A internal/app/transport_observability_test.go +82 -0

@@ -0,0 +1,82 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/json"
	6	+	"os"
	7	+	"path/filepath"
	8	+	"strings"
	9	+	"testing"
	10	+	"time"
	11	+
	12	+	"github.com/bnema/vev/internal/ports"
	13	+	portsmocks "github.com/bnema/vev/internal/ports/mocks"
	14	+	)
	15	+
	16	+	func TestPerformanceTraceUsesHarnessProcessMapping(t *testing.T) {
	17	+	path := filepath.Join(t.TempDir(), "process.jsonl")
	18	+	t.Setenv("VEV_PERF_TRACE", path)
	19	+	t.Setenv("VEV_PERF_PROCESS_ID", "idle-local-r001-daemon-01")
	20	+	t.Setenv("VEV_PERF_SCENARIO", "1x4-idle-local")
	21	+	t.Setenv("VEV_PERF_RUN", "1")
	22	+
	23	+	clock := portsmocks.NewMockClock(t)
	24	+	clock.EXPECT().Now().Return(time.Unix(0, 1)).Once()
	25	+	observer, closer, err := performanceTrace(clock)
	26	+	if err != nil {
	27	+	t.Fatal(err)
	28	+	}
	29	+	if observer == nil \|\| closer == nil {
	30	+	t.Fatal("performance trace was not configured")
	31	+	}
	32	+	if _, ok := observer.(ports.SerializedRuntimeObserver); !ok {
	33	+	t.Fatalf("performance trace observer = %T, want serialized owner", observer)
	34	+	}
	35	+	observer.ObserveRuntime(ports.RuntimeMark{Schema: ports.RuntimeMarkSchema, Component: "daemon", Scenario: "runtime", Run: 1, Kind: ports.RuntimeEmitEnd, Valid: true})
	36	+	if err := closer.Close(); err != nil {
	37	+	t.Fatal(err)
	38	+	}
	39	+	if err := closer.Close(); err != nil {
	40	+	t.Fatal(err)
	41	+	}
	42	+	f, err := os.Open(path)
	43	+	if err != nil {
	44	+	t.Fatal(err)
	45	+	}
	46	+	defer f.Close()
	47	+	var mark struct {
	48	+	ProcessID string `json:"process_id"`
	49	+	Scenario string `json:"scenario"`
	50	+	Run uint64 `json:"run"`
	51	+	Sequence uint64 `json:"sequence"`
	52	+	RequestID uint64 `json:"request_id"`
	53	+	Epoch uint64 `json:"epoch"`
	54	+	}
	55	+	if err := json.NewDecoder(bufio.NewReader(f)).Decode(&mark); err != nil {
	56	+	t.Fatal(err)
	57	+	}
	58	+	if mark.ProcessID != "idle-local-r001-daemon-01" \|\| mark.Scenario != "1x4-idle-local" \|\| mark.Run != 1 \|\| mark.Sequence == 0 \|\| mark.RequestID == 0 \|\| mark.Epoch == 0 {
	59	+	t.Fatalf("mark does not match harness mapping: %+v", mark)
	60	+	}
	61	+	}
	62	+
	63	+	func TestPerformanceTraceAppWiresOneObserverPerProcess(t *testing.T) {
	64	+	source, err := os.ReadFile("run.go")
	65	+	if err != nil {
	66	+	t.Fatalf("ReadFile(run.go): %v", err)
	67	+	}
	68	+	text := string(source)
	69	+	for _, required := range []string{
	70	+	"VEV_PERF_TRACE",
	71	+	"VEV_PERF_PROCESS_ID",
	72	+	"observability.NewJSONL",
	73	+	"WithRuntimeObserver",
	74	+	} {
	75	+	if !strings.Contains(text, required) {
	76	+	t.Errorf("runtime trace wiring missing %q", required)
	77	+	}
	78	+	}
	79	+	if strings.Contains(text, "WithRuntimeObserver(clock") \|\| strings.Contains(text, "WithRuntimeObserver(clk") {
	80	+	t.Error("consumer observer configuration must not accept a clock")
	81	+	}
	82	+	}

M internal/ports/mocks/mocks.go +67 -0


@@ -3129,3 +3129,70 @@ func (_c *MockRemoteDialerFactory_DialerForRemote_Call) RunAndReturn(run func(ta
3129	3129		_c.Call.Return(run)
3130	3130		return _c
3131	3131		}
	3132	+
	3133	+	// NewMockRuntimeObserver creates a new instance of MockRuntimeObserver. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	3134	+	// The first argument is typically a *testing.T value.
	3135	+	func NewMockRuntimeObserver(t interface {
	3136	+	mock.TestingT
	3137	+	Cleanup(func())
	3138	+	}) *MockRuntimeObserver {
	3139	+	mock := &MockRuntimeObserver{}
	3140	+	mock.Mock.Test(t)
	3141	+
	3142	+	t.Cleanup(func() { mock.AssertExpectations(t) })
	3143	+
	3144	+	return mock
	3145	+	}
	3146	+
	3147	+	// MockRuntimeObserver is an autogenerated mock type for the RuntimeObserver type
	3148	+	type MockRuntimeObserver struct {
	3149	+	mock.Mock
	3150	+	}
	3151	+
	3152	+	type MockRuntimeObserver_Expecter struct {
	3153	+	mock *mock.Mock
	3154	+	}
	3155	+
	3156	+	func (_m MockRuntimeObserver) EXPECT() MockRuntimeObserver_Expecter {
	3157	+	return &MockRuntimeObserver_Expecter{mock: &_m.Mock}
	3158	+	}
	3159	+
	3160	+	// ObserveRuntime provides a mock function for the type MockRuntimeObserver
	3161	+	func (_mock *MockRuntimeObserver) ObserveRuntime(runtimeMark ports.RuntimeMark) {
	3162	+	_mock.Called(runtimeMark)
	3163	+	return
	3164	+	}
	3165	+
	3166	+	// MockRuntimeObserver_ObserveRuntime_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ObserveRuntime'
	3167	+	type MockRuntimeObserver_ObserveRuntime_Call struct {
	3168	+	*mock.Call
	3169	+	}
	3170	+
	3171	+	// ObserveRuntime is a helper method to define mock.On call
	3172	+	// - runtimeMark ports.RuntimeMark
	3173	+	func (_e MockRuntimeObserver_Expecter) ObserveRuntime(runtimeMark any) MockRuntimeObserver_ObserveRuntime_Call {
	3174	+	return &MockRuntimeObserver_ObserveRuntime_Call{Call: _e.mock.On("ObserveRuntime", runtimeMark)}
	3175	+	}
	3176	+
	3177	+	func (_c MockRuntimeObserver_ObserveRuntime_Call) Run(run func(runtimeMark ports.RuntimeMark)) MockRuntimeObserver_ObserveRuntime_Call {
	3178	+	_c.Call.Run(func(args mock.Arguments) {
	3179	+	var arg0 ports.RuntimeMark
	3180	+	if args[0] != nil {
	3181	+	arg0 = args[0].(ports.RuntimeMark)
	3182	+	}
	3183	+	run(
	3184	+	arg0,
	3185	+	)
	3186	+	})
	3187	+	return _c
	3188	+	}
	3189	+
	3190	+	func (_c MockRuntimeObserver_ObserveRuntime_Call) Return() MockRuntimeObserver_ObserveRuntime_Call {
	3191	+	_c.Call.Return()
	3192	+	return _c
	3193	+	}
	3194	+
	3195	+	func (_c MockRuntimeObserver_ObserveRuntime_Call) RunAndReturn(run func(runtimeMark ports.RuntimeMark)) MockRuntimeObserver_ObserveRuntime_Call {
	3196	+	_c.Run(run)
	3197	+	return _c
	3198	+	}

A internal/ports/runtime_observer.go +297 -0

@@ -0,0 +1,297 @@
	1	+	package ports
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"sync"
	6	+	"sync/atomic"
	7	+	)
	8	+
	9	+	// RuntimeMarkSchema is the version of the process-local performance trace.
	10	+	const RuntimeMarkSchema uint16 = 1
	11	+
	12	+	// RuntimeMarkKind identifies a process-local runtime boundary. It deliberately
	13	+	// contains no protocol payload, cells, or rendering policy.
	14	+	type RuntimeMarkKind string
	15	+
	16	+	const (
	17	+	RuntimeInputInjected RuntimeMarkKind = "input_injected"
	18	+	RuntimeResizeRequested RuntimeMarkKind = "resize_requested"
	19	+	RuntimeResizeCommitted RuntimeMarkKind = "resize_committed"
	20	+	RuntimeCaptureStart RuntimeMarkKind = "capture_start"
	21	+	RuntimeCaptureEnd RuntimeMarkKind = "capture_end"
	22	+	RuntimeComposeStart RuntimeMarkKind = "compose_start"
	23	+	RuntimeComposeEnd RuntimeMarkKind = "compose_end"
	24	+	RuntimeDiffStart RuntimeMarkKind = "diff_start"
	25	+	RuntimeDiffEnd RuntimeMarkKind = "diff_end"
	26	+	RuntimeQueueEnqueued RuntimeMarkKind = "queue_enqueued"
	27	+	RuntimeQueueDequeued RuntimeMarkKind = "queue_dequeued"
	28	+	RuntimeACKBlockedStart RuntimeMarkKind = "ack_blocked_start"
	29	+	RuntimeACKBlockedEnd RuntimeMarkKind = "ack_blocked_end"
	30	+	RuntimeEmitStart RuntimeMarkKind = "emit_start"
	31	+	RuntimeEmitEnd RuntimeMarkKind = "emit_end"
	32	+	RuntimeAdapterSendStart RuntimeMarkKind = "adapter_send_start"
	33	+	RuntimeAdapterSendEnd RuntimeMarkKind = "adapter_send_end"
	34	+	RuntimeAdapterReceiveStart RuntimeMarkKind = "adapter_receive_start"
	35	+	RuntimeAdapterReceiveEnd RuntimeMarkKind = "adapter_receive_end"
	36	+	RuntimeTerminalFlushed RuntimeMarkKind = "terminal_flushed"
	37	+	RuntimeTransportDiagnostic RuntimeMarkKind = "transport_diagnostic"
	38	+	)
	39	+
	40	+	// RuntimeMark is a process-local observation record. Tick is intentionally
	41	+	// left zero by producers: the concrete process observer owns timestamping.
	42	+	type RuntimeMark struct {
	43	+	Schema uint16
	44	+	ProcessID string
	45	+	Component string
	46	+	Scenario string
	47	+	Run uint64
	48	+	Sequence uint64
	49	+	RequestID uint64
	50	+	Epoch uint64
	51	+	Kind RuntimeMarkKind
	52	+	Tick int64
	53	+	Bytes uint64
	54	+	Fragments uint64
	55	+	Retransmits uint64
	56	+	Pending uint64
	57	+	AckRTTNanos int64
	58	+	Valid bool
	59	+	}
	60	+
	61	+	// RuntimeObserver receives process-local marks. Implementations must not let
	62	+	// a producer provide a timestamp in another clock domain.
	63	+	type RuntimeObserver interface{ ObserveRuntime(RuntimeMark) }
	64	+
	65	+	// SerializedRuntimeObserver accepts marks without waiting for observer I/O.
	66	+	// Flush and Close wait until every accepted mark has reached Observer. Close
	67	+	// is idempotent and rejects later marks.
	68	+	type SerializedRuntimeObserver interface {
	69	+	RuntimeObserver
	70	+	Flush()
	71	+	Close()
	72	+	}
	73	+
	74	+	// NewSerializedRuntimeObserver isolates producers from a potentially slow
	75	+	// observer with one ordered worker. capacity bounds queued marks. If the queue
	76	+	// fills, dropped marks are represented by one ordered diagnostic mark before
	77	+	// reporting resumes, so a trace can never silently claim completeness.
	78	+	func NewSerializedRuntimeObserver(observer RuntimeObserver, capacity int) SerializedRuntimeObserver {
	79	+	if observer == nil {
	80	+	return nil
	81	+	}
	82	+	if capacity < 1 {
	83	+	capacity = 1
	84	+	}
	85	+	o := &serializedRuntimeObserver{observer: observer, capacity: capacity, done: make(chan struct{})}
	86	+	o.cond = sync.NewCond(&o.mu)
	87	+	go o.run()
	88	+	return o
	89	+	}
	90	+
	91	+	// EnsureSerializedRuntimeObserver returns observer's existing serialized
	92	+	// owner when it has one. Otherwise it creates an owned worker. Callers close
	93	+	// the result only when owned is true, preventing nested workers and duplicate
	94	+	// lifecycle ownership when application wiring already serialized the process
	95	+	// observer.
	96	+	func EnsureSerializedRuntimeObserver(observer RuntimeObserver, capacity int) (serialized SerializedRuntimeObserver, owned bool) {
	97	+	if observer == nil {
	98	+	return nil, false
	99	+	}
	100	+	if serialized, ok := observer.(SerializedRuntimeObserver); ok {
	101	+	return serialized, false
	102	+	}
	103	+	return NewSerializedRuntimeObserver(observer, capacity), true
	104	+	}
	105	+
	106	+	type serializedRuntimeObserver struct {
	107	+	observer RuntimeObserver
	108	+	capacity int
	109	+
	110	+	mu sync.Mutex
	111	+	cond *sync.Cond
	112	+	marks []RuntimeMark
	113	+	dropped uint64
	114	+	closed bool
	115	+	active bool
	116	+	done chan struct{}
	117	+	}
	118	+
	119	+	func (o *serializedRuntimeObserver) ObserveRuntime(mark RuntimeMark) {
	120	+	if o == nil {
	121	+	return
	122	+	}
	123	+	o.mu.Lock()
	124	+	defer o.mu.Unlock()
	125	+	if o.closed {
	126	+	return
	127	+	}
	128	+	if o.dropped != 0 {
	129	+	// Reserve space for both the gap marker and the first post-gap mark.
	130	+	// This preserves the producer order around a bounded-queue loss.
	131	+	if len(o.marks)+2 > o.capacity {
	132	+	o.dropped++
	133	+	return
	134	+	}
	135	+	o.marks = append(o.marks, runtimeObserverGapMark(o.dropped), mark)
	136	+	o.dropped = 0
	137	+	o.cond.Signal()
	138	+	return
	139	+	}
	140	+	if len(o.marks) == o.capacity {
	141	+	o.dropped = 1
	142	+	return
	143	+	}
	144	+	o.marks = append(o.marks, mark)
	145	+	o.cond.Signal()
	146	+	}
	147	+
	148	+	func runtimeObserverGapMark(dropped uint64) RuntimeMark {
	149	+	return NewRuntimeMark("observer", RuntimeTransportDiagnostic, dropped, false)
	150	+	}
	151	+
	152	+	func (o *serializedRuntimeObserver) Flush() {
	153	+	if o == nil {
	154	+	return
	155	+	}
	156	+	o.mu.Lock()
	157	+	for len(o.marks) != 0 \|\| o.active \|\| o.dropped != 0 {
	158	+	o.cond.Signal()
	159	+	o.cond.Wait()
	160	+	}
	161	+	closed := o.closed
	162	+	o.mu.Unlock()
	163	+	if closed {
	164	+	<-o.done
	165	+	}
	166	+	}
	167	+
	168	+	func (o *serializedRuntimeObserver) Close() {
	169	+	if o == nil {
	170	+	return
	171	+	}
	172	+	o.mu.Lock()
	173	+	if !o.closed {
	174	+	o.closed = true
	175	+	o.cond.Broadcast()
	176	+	}
	177	+	o.mu.Unlock()
	178	+	<-o.done
	179	+	}
	180	+
	181	+	func (o *serializedRuntimeObserver) run() {
	182	+	defer close(o.done)
	183	+	for {
	184	+	o.mu.Lock()
	185	+	for len(o.marks) == 0 && o.dropped == 0 && !o.closed {
	186	+	o.cond.Wait()
	187	+	}
	188	+	if len(o.marks) == 0 && o.dropped != 0 {
	189	+	o.marks = append(o.marks, runtimeObserverGapMark(o.dropped))
	190	+	o.dropped = 0
	191	+	}
	192	+	if len(o.marks) == 0 && o.closed {
	193	+	o.cond.Broadcast()
	194	+	o.mu.Unlock()
	195	+	return
	196	+	}
	197	+	mark := o.marks[0]
	198	+	o.marks = o.marks[1:]
	199	+	o.active = true
	200	+	o.mu.Unlock()
	201	+
	202	+	o.observer.ObserveRuntime(mark)
	203	+
	204	+	o.mu.Lock()
	205	+	o.active = false
	206	+	o.cond.Broadcast()
	207	+	o.mu.Unlock()
	208	+	}
	209	+	}
	210	+
	211	+	// RuntimeMarkValid reports whether a mark has all correlation fields required
	212	+	// by the JSONL schema. Tick is checked separately by the timestamp owner.
	213	+	func RuntimeMarkValid(mark RuntimeMark) bool {
	214	+	if mark.Schema != RuntimeMarkSchema \|\| mark.Component == "" \|\| mark.Scenario == "" \|\| mark.Run == 0 \|\| mark.Sequence == 0 \|\| mark.RequestID == 0 \|\| mark.Epoch == 0 {
	215	+	return false
	216	+	}
	217	+	switch mark.Kind {
	218	+	case RuntimeInputInjected, RuntimeResizeRequested, RuntimeResizeCommitted,
	219	+	RuntimeCaptureStart, RuntimeCaptureEnd, RuntimeComposeStart, RuntimeComposeEnd,
	220	+	RuntimeDiffStart, RuntimeDiffEnd, RuntimeQueueEnqueued, RuntimeQueueDequeued,
	221	+	RuntimeACKBlockedStart, RuntimeACKBlockedEnd, RuntimeEmitStart, RuntimeEmitEnd,
	222	+	RuntimeAdapterSendStart, RuntimeAdapterSendEnd, RuntimeAdapterReceiveStart,
	223	+	RuntimeAdapterReceiveEnd, RuntimeTerminalFlushed, RuntimeTransportDiagnostic:
	224	+	return true
	225	+	default:
	226	+	return false
	227	+	}
	228	+	}
	229	+
	230	+	// RuntimeCorrelation identifies one attempted operation. It is deliberately
	231	+	// copied to both endpoints of a span; process ID remains owned by the sink.
	232	+	type RuntimeCorrelation struct {
	233	+	Scenario string
	234	+	Run, Sequence, RequestID, Epoch uint64
	235	+	}
	236	+
	237	+	// RuntimeCorrelationInputs are supplied once by process orchestration. They
	238	+	// are deliberately data-only: consumer APIs continue to receive only their
	239	+	// one RuntimeObserver argument and never a clock or tick.
	240	+	type RuntimeCorrelationInputs struct {
	241	+	Scenario string
	242	+	Run uint64
	243	+	}
	244	+
	245	+	// NewRuntimeCorrelationObserver binds every mark from one OS process to its
	246	+	// harness manifest scenario/run. Operation IDs remain producer-owned when
	247	+	// supplied; missing IDs are assigned from the process-wide allocator.
	248	+	func NewRuntimeCorrelationObserver(observer RuntimeObserver, inputs RuntimeCorrelationInputs) (RuntimeObserver, error) {
	249	+	if observer == nil \|\| inputs.Scenario == "" \|\| inputs.Run == 0 {
	250	+	return nil, errors.New("invalid runtime correlation inputs")
	251	+	}
	252	+	return &runtimeCorrelationObserver{observer: observer, inputs: inputs}, nil
	253	+	}
	254	+
	255	+	type runtimeCorrelationObserver struct {
	256	+	observer RuntimeObserver
	257	+	inputs RuntimeCorrelationInputs
	258	+	}
	259	+
	260	+	func (o *runtimeCorrelationObserver) ObserveRuntime(mark RuntimeMark) {
	261	+	if o == nil \|\| o.observer == nil {
	262	+	return
	263	+	}
	264	+	mark.Scenario, mark.Run = o.inputs.Scenario, o.inputs.Run
	265	+	if mark.Sequence == 0 {
	266	+	mark.Sequence = runtimeMarkSequence.Add(1)
	267	+	}
	268	+	if mark.RequestID == 0 {
	269	+	mark.RequestID = mark.Sequence
	270	+	}
	271	+	if mark.Epoch == 0 {
	272	+	mark.Epoch = mark.RequestID
	273	+	}
	274	+	o.observer.ObserveRuntime(mark)
	275	+	}
	276	+
	277	+	var runtimeMarkSequence atomic.Uint64
	278	+
	279	+	// NewRuntimeCorrelation makes a unique, process-local operation identity.
	280	+	// Components with a domain request/epoch should use those values directly via
	281	+	// NewRuntimeMarkWithCorrelation instead of inventing a second identity.
	282	+	func NewRuntimeCorrelation() RuntimeCorrelation {
	283	+	sequence := runtimeMarkSequence.Add(1)
	284	+	return RuntimeCorrelation{Scenario: "runtime", Run: 1, Sequence: sequence, RequestID: sequence, Epoch: sequence}
	285	+	}
	286	+
	287	+	// NewRuntimeMarkWithCorrelation returns a timestamp-free mark carrying the
	288	+	// supplied operation identity. Start/end callers must reuse correlation.
	289	+	func NewRuntimeMarkWithCorrelation(component string, correlation RuntimeCorrelation, kind RuntimeMarkKind, bytes uint64, valid bool) RuntimeMark {
	290	+	return RuntimeMark{Schema: RuntimeMarkSchema, Component: component, Scenario: correlation.Scenario, Run: correlation.Run, Sequence: correlation.Sequence, RequestID: correlation.RequestID, Epoch: correlation.Epoch, Kind: kind, Bytes: bytes, Valid: valid}
	291	+	}
	292	+
	293	+	// NewRuntimeMark is retained for one-shot marks. Unlike the old helper it
	294	+	// never returns constant correlation IDs.
	295	+	func NewRuntimeMark(component string, kind RuntimeMarkKind, bytes uint64, valid bool) RuntimeMark {
	296	+	return NewRuntimeMarkWithCorrelation(component, NewRuntimeCorrelation(), kind, bytes, valid)
	297	+	}

A internal/ports/runtime_observer_test.go +204 -0

@@ -0,0 +1,204 @@
	1	+	package ports
	2	+
	3	+	import (
	4	+	"reflect"
	5	+	"sync"
	6	+	"testing"
	7	+	"time"
	8	+	)
	9	+
	10	+	func TestRuntimeObserverContract(t *testing.T) {
	11	+	wantFields := []struct {
	12	+	name string
	13	+	typ reflect.Type
	14	+	}{
	15	+	{"Schema", reflect.TypeFor[uint16]()},
	16	+	{"ProcessID", reflect.TypeFor[string]()},
	17	+	{"Component", reflect.TypeFor[string]()},
	18	+	{"Scenario", reflect.TypeFor[string]()},
	19	+	{"Run", reflect.TypeFor[uint64]()},
	20	+	{"Sequence", reflect.TypeFor[uint64]()},
	21	+	{"RequestID", reflect.TypeFor[uint64]()},
	22	+	{"Epoch", reflect.TypeFor[uint64]()},
	23	+	{"Kind", reflect.TypeFor[RuntimeMarkKind]()},
	24	+	{"Tick", reflect.TypeFor[int64]()},
	25	+	{"Bytes", reflect.TypeFor[uint64]()},
	26	+	{"Fragments", reflect.TypeFor[uint64]()},
	27	+	{"Retransmits", reflect.TypeFor[uint64]()},
	28	+	{"Pending", reflect.TypeFor[uint64]()},
	29	+	{"AckRTTNanos", reflect.TypeFor[int64]()},
	30	+	{"Valid", reflect.TypeFor[bool]()},
	31	+	}
	32	+
	33	+	typ := reflect.TypeFor[RuntimeMark]()
	34	+	if typ.NumField() != len(wantFields) {
	35	+	t.Fatalf("RuntimeMark has %d fields, want exactly %d", typ.NumField(), len(wantFields))
	36	+	}
	37	+	for i, want := range wantFields {
	38	+	field := typ.Field(i)
	39	+	if field.Name != want.name \|\| field.Type != want.typ {
	40	+	t.Fatalf("RuntimeMark field %d = %s %v, want %s %v", i, field.Name, field.Type, want.name, want.typ)
	41	+	}
	42	+	}
	43	+
	44	+	var _ RuntimeObserver = runtimeObserverFunc(func(RuntimeMark) {})
	45	+	}
	46	+
	47	+	func TestSerializedRuntimeObserverFlushesInOrderAndCloses(t *testing.T) {
	48	+	var mu sync.Mutex
	49	+	var got []RuntimeMark
	50	+	observer := NewSerializedRuntimeObserver(runtimeObserverFunc(func(mark RuntimeMark) {
	51	+	mu.Lock()
	52	+	got = append(got, mark)
	53	+	mu.Unlock()
	54	+	}), 2)
	55	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeEmitStart, 1, true))
	56	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeEmitEnd, 1, true))
	57	+	observer.Flush()
	58	+	observer.Close()
	59	+	observer.Close()
	60	+
	61	+	mu.Lock()
	62	+	defer mu.Unlock()
	63	+	if len(got) != 2 \|\| got[0].Kind != RuntimeEmitStart \|\| got[1].Kind != RuntimeEmitEnd {
	64	+	t.Fatalf("serialized marks = %#v", got)
	65	+	}
	66	+	}
	67	+
	68	+	func TestEnsureSerializedRuntimeObserverReusesExistingOwner(t *testing.T) {
	69	+	existing := NewSerializedRuntimeObserver(runtimeObserverFunc(func(RuntimeMark) {}), 1)
	70	+	reporter, owned := EnsureSerializedRuntimeObserver(existing, 1)
	71	+	if owned \|\| reporter != existing {
	72	+	t.Fatalf("EnsureSerializedRuntimeObserver() = (%T, %t), want existing unowned reporter", reporter, owned)
	73	+	}
	74	+	existing.Close()
	75	+	}
	76	+
	77	+	func TestSerializedRuntimeObserverReportsBoundedQueueLoss(t *testing.T) {
	78	+	entered := make(chan struct{})
	79	+	release := make(chan struct{})
	80	+	var mu sync.Mutex
	81	+	var got []RuntimeMark
	82	+	observer := NewSerializedRuntimeObserver(runtimeObserverFunc(func(mark RuntimeMark) {
	83	+	if mark.Kind == RuntimeEmitStart {
	84	+	close(entered)
	85	+	<-release
	86	+	}
	87	+	mu.Lock()
	88	+	got = append(got, mark)
	89	+	mu.Unlock()
	90	+	}), 2)
	91	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeEmitStart, 1, true))
	92	+	select {
	93	+	case <-entered:
	94	+	case <-time.After(time.Second):
	95	+	t.Fatal("observer did not start")
	96	+	}
	97	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeComposeStart, 1, true))
	98	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeComposeEnd, 1, true))
	99	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeEmitEnd, 1, true)) // bounded loss
	100	+	close(release)
	101	+	observer.Flush()
	102	+	observer.Close()
	103	+
	104	+	mu.Lock()
	105	+	defer mu.Unlock()
	106	+	if len(got) != 4 \|\| got[0].Kind != RuntimeEmitStart \|\| got[1].Kind != RuntimeComposeStart \|\| got[2].Kind != RuntimeComposeEnd \|\| got[3].Kind != RuntimeTransportDiagnostic \|\| got[3].Bytes != 1 \|\| got[3].Valid {
	107	+	t.Fatalf("bounded-loss marks = %#v", got)
	108	+	}
	109	+	}
	110	+
	111	+	func TestRuntimeObserverRequiredSpanKinds(t *testing.T) {
	112	+	pairs := []struct {
	113	+	start RuntimeMarkKind
	114	+	end RuntimeMarkKind
	115	+	}{
	116	+	{RuntimeDiffStart, RuntimeDiffEnd},
	117	+	{RuntimeQueueEnqueued, RuntimeQueueDequeued},
	118	+	{RuntimeACKBlockedStart, RuntimeACKBlockedEnd},
	119	+	{RuntimeAdapterSendStart, RuntimeAdapterSendEnd},
	120	+	{RuntimeAdapterReceiveStart, RuntimeAdapterReceiveEnd},
	121	+	}
	122	+	for _, pair := range pairs {
	123	+	if pair.start == "" \|\| pair.end == "" \|\| pair.start == pair.end {
	124	+	t.Fatalf("invalid required span mapping %#v", pair)
	125	+	}
	126	+	}
	127	+	}
	128	+
	129	+	func TestRuntimeCorrelationObserverUsesHarnessProcessInputs(t *testing.T) {
	130	+	var got []RuntimeMark
	131	+	observer, err := NewRuntimeCorrelationObserver(runtimeObserverFunc(func(mark RuntimeMark) {
	132	+	got = append(got, mark)
	133	+	}), RuntimeCorrelationInputs{Scenario: "1x4-idle-local", Run: 7})
	134	+	if err != nil {
	135	+	t.Fatal(err)
	136	+	}
	137	+
	138	+	observer.ObserveRuntime(RuntimeMark{Schema: RuntimeMarkSchema, Component: "daemon", Scenario: "runtime", Run: 1, Kind: RuntimeEmitEnd, Valid: true})
	139	+	observer.ObserveRuntime(RuntimeMark{Schema: RuntimeMarkSchema, Component: "daemon", Scenario: "runtime", Run: 1, Sequence: 9, RequestID: 10, Epoch: 11, Kind: RuntimeEmitStart, Valid: true})
	140	+
	141	+	if len(got) != 2 {
	142	+	t.Fatalf("marks=%+v", got)
	143	+	}
	144	+	for _, mark := range got {
	145	+	if mark.Scenario != "1x4-idle-local" \|\| mark.Run != 7 {
	146	+	t.Fatalf("harness correlation was not applied: %+v", mark)
	147	+	}
	148	+	if mark.Sequence == 0 \|\| mark.RequestID == 0 \|\| mark.Epoch == 0 {
	149	+	t.Fatalf("missing deterministic operation identity: %+v", mark)
	150	+	}
	151	+	}
	152	+	if got[1].Sequence != 9 \|\| got[1].RequestID != 10 \|\| got[1].Epoch != 11 {
	153	+	t.Fatalf("explicit operation identity changed: %+v", got[1])
	154	+	}
	155	+	}
	156	+
	157	+	func TestRuntimeCorrelationObserverAllocatesMissingIDsProcessWide(t *testing.T) {
	158	+	var got []RuntimeMark
	159	+	collect := runtimeObserverFunc(func(mark RuntimeMark) { got = append(got, mark) })
	160	+	first, err := NewRuntimeCorrelationObserver(collect, RuntimeCorrelationInputs{Scenario: "scenario", Run: 1})
	161	+	if err != nil {
	162	+	t.Fatal(err)
	163	+	}
	164	+	second, err := NewRuntimeCorrelationObserver(collect, RuntimeCorrelationInputs{Scenario: "scenario", Run: 1})
	165	+	if err != nil {
	166	+	t.Fatal(err)
	167	+	}
	168	+
	169	+	first.ObserveRuntime(RuntimeMark{Schema: RuntimeMarkSchema, Component: "first", Kind: RuntimeEmitStart, Valid: true})
	170	+	second.ObserveRuntime(RuntimeMark{Schema: RuntimeMarkSchema, Component: "second", Kind: RuntimeEmitEnd, Valid: true})
	171	+
	172	+	if len(got) != 2 {
	173	+	t.Fatalf("marks=%+v", got)
	174	+	}
	175	+	if got[0].Sequence == got[1].Sequence {
	176	+	t.Fatalf("wrapper-local sequence collision: %+v", got)
	177	+	}
	178	+	for _, mark := range got {
	179	+	if mark.RequestID != mark.Sequence \|\| mark.Epoch != mark.RequestID {
	180	+	t.Fatalf("missing-ID fallback changed: %+v", mark)
	181	+	}
	182	+	}
	183	+	}
	184	+
	185	+	func TestRuntimeCorrelationObserverRejectsInvalidHarnessInputs(t *testing.T) {
	186	+	for _, in := range []RuntimeCorrelationInputs{{}, {Scenario: "scenario"}} {
	187	+	if _, err := NewRuntimeCorrelationObserver(runtimeObserverFunc(func(RuntimeMark) {}), in); err == nil {
	188	+	t.Fatalf("invalid inputs accepted: %+v", in)
	189	+	}
	190	+	}
	191	+	}
	192	+
	193	+	func TestRuntimeMarkHasNoRenderingPayloadOrPolicyFields(t *testing.T) {
	194	+	typ := reflect.TypeFor[RuntimeMark]()
	195	+	for _, forbidden := range []string{"Cell", "Cells", "Payload", "Frame", "Renderer", "Policy", "Transport"} {
	196	+	if _, ok := typ.FieldByName(forbidden); ok {
	197	+	t.Fatalf("RuntimeMark must not carry %s", forbidden)
	198	+	}
	199	+	}
	200	+	}
	201	+
	202	+	type runtimeObserverFunc func(RuntimeMark)
	203	+
	204	+	func (f runtimeObserverFunc) ObserveRuntime(mark RuntimeMark) { f(mark) }

M internal/usecase/client/client.go +52 -5


@@ -102,6 +102,10 @@ const stdinBufSize = 4096
102	102		// goroutine, decoupling the input/resize pumps from transport back-pressure.
103	103		const sendQueueDepth = 64
104	104
	105	+	// runtimeObserverQueueDepth bounds marks awaiting the one reporting worker.
	106	+	// A full queue records an ordered diagnostic gap rather than blocking ACKs.
	107	+	const runtimeObserverQueueDepth = 64
	108	+
105	109		var reconnectSleep = sleepReconnect
106	110		var reconnectSleepWithResize = sleepReconnectWithResizeEvents
107	111

@@ -120,7 +124,36 @@ const (
120	124		// ClipboardReader configured (clipboard non-nil) — local attaches forward
121	125		// 0x16 untouched so the locally running agent's own clipboard handling
122	126		// applies.
123		-	func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger) (retErr error) {
	127	+	// Option configures opt-in client runtime observation.
	128	+	type Option func(*runtimeOptions)
	129	+
	130	+	type runtimeOptions struct{ observer ports.RuntimeObserver }
	131	+
	132	+	// WithRuntimeObserver enables process-local marks. It intentionally accepts no
	133	+	// clock; timestamps are assigned by the concrete observer.
	134	+	func WithRuntimeObserver(observer ports.RuntimeObserver) Option {
	135	+	return func(opts *runtimeOptions) { opts.observer = observer }
	136	+	}
	137	+
	138	+	func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger) error {
	139	+	return run(ctx, dialer, term, clk, intent, name, remote, clipboard, log, nil)
	140	+	}
	141	+
	142	+	// RunWithRuntimeObserver is the application wiring entry point.
	143	+	func RunWithRuntimeObserver(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger, observer ports.RuntimeObserver) error {
	144	+	return run(ctx, dialer, term, clk, intent, name, remote, clipboard, log, observer)
	145	+	}
	146	+
	147	+	func run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger, observer ports.RuntimeObserver) (retErr error) {
	148	+	if reporter, owned := ports.EnsureSerializedRuntimeObserver(observer, runtimeObserverQueueDepth); reporter != nil {
	149	+	if owned {
	150	+	defer func() {
	151	+	reporter.Flush()
	152	+	reporter.Close()
	153	+	}()
	154	+	}
	155	+	observer = reporter
	156	+	}
124	157		if log == nil {
125	158		log = slog.Default()
126	159		}

@@ -244,7 +277,7 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port
244	277		if reporter, ok := transport.(ports.LinkStateReporter); ok {
245	278		linkEvents = reporter.LinkEvents()
246	279		}
247		-	result := attachOnce(ctx, transport, term, clk, attemptIntent, attemptName, resumeToken, processClientID, &ms, enterRaw, clearStatus, drawStatusStage, linkEvents, remote, clipboard, log)
	280	+	result := attachOnce(ctx, transport, term, clk, attemptIntent, attemptName, resumeToken, processClientID, &ms, enterRaw, clearStatus, drawStatusStage, linkEvents, remote, clipboard, log, observer)
248	281		if result.welcomed {
249	282		backoff = defaultReconnectBackoff.initial
250	283		}

@@ -311,8 +344,14 @@ func sleepReconnectWithResizeEvents(ctx context.Context, clk ports.Clock, d time
311	344		// disappears, or the context is cancelled.
312	345		//
313	346		// It is kept as a compatibility wrapper for callers that still own dialing.
314		-	func Attach(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string) error {
315		-	return Run(ctx, singleTransportDialer{transport: transport}, term, clk, intent, name, false, nil, slog.Default())
	347	+	func Attach(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, options ...Option) error {
	348	+	var opts runtimeOptions
	349	+	for _, option := range options {
	350	+	if option != nil {
	351	+	option(&opts)
	352	+	}
	353	+	}
	354	+	return run(ctx, singleTransportDialer{transport: transport}, term, clk, intent, name, false, nil, slog.Default(), opts.observer)
316	355		}
317	356
318	357		type singleTransportDialer struct{ transport ports.Transport }

@@ -346,7 +385,11 @@ func requestedOutputWindow(transport ports.Transport) uint8 {
346	385		return 8
347	386		}
348	387
349		-	func attachOnce(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, resumeToken uint64, clientID [16]byte, ms milestones, enterRaw func() error, clearStatus func(), drawStatusStage func(reconnectStage), linkEvents <-chan ports.LinkEvent, remote bool, clipboard ports.ClipboardReader, log slog.Logger) attachResult {
	388	+	func attachOnce(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, resumeToken uint64, clientID [16]byte, ms milestones, enterRaw func() error, clearStatus func(), drawStatusStage func(reconnectStage), linkEvents <-chan ports.LinkEvent, remote bool, clipboard ports.ClipboardReader, log slog.Logger, observers ...ports.RuntimeObserver) attachResult {
	389	+	var observer ports.RuntimeObserver
	390	+	if len(observers) != 0 {
	391	+	observer = observers[0]
	392	+	}
350	393		// 1. Handshake: send Hello with our size and TERM.
351	394		size, err := term.Size()
352	395		if err != nil {

@@ -501,6 +544,10 @@ func attachOnce(ctx context.Context, transport ports.Transport, term ports.Termi
501	544		if ferr := term.Flush(); ferr != nil {
502	545		return welcomedResult(fmt.Errorf("vev: flushing terminal: %w", ferr))
503	546		}
	547	+	// The terminal boundary is after a successful flush and before ACK.
	548	+	if observer != nil {
	549	+	observer.ObserveRuntime(ports.NewRuntimeMark("client", ports.RuntimeTerminalFlushed, uint64(len(o.Data)), true))
	550	+	}
504	551		if o.NewStateNum != 0 {
505	552		ackQueue.offer(o.NewStateNum)
506	553		}

A internal/usecase/client/transport_observability_test.go +140 -0

@@ -0,0 +1,140 @@
	1	+	package client_test
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"context"
	6	+	"sync"
	7	+	"sync/atomic"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/stretchr/testify/require"
	12	+
	13	+	"github.com/bnema/vev/internal/domain"
	14	+	"github.com/bnema/vev/internal/ports"
	15	+	portsmocks "github.com/bnema/vev/internal/ports/mocks"
	16	+	"github.com/bnema/vev/internal/usecase/client"
	17	+	)
	18	+
	19	+	func TestBlockingRuntimeObserverDoesNotDelayTerminalFlushOrACK(t *testing.T) {
	20	+	var out bytes.Buffer
	21	+	var restores atomic.Int32
	22	+	resizeCh := make(chan domain.Size)
	23	+	flushed := make(chan struct{})
	24	+	term := portsmocks.NewMockTerminal(t)
	25	+	term.EXPECT().Size().Return(domain.Size{Cols: 80, Rows: 24}, nil).Once()
	26	+	term.EXPECT().EnterRaw().Return(func() error { restores.Add(1); return nil }, nil).Once()
	27	+	in := newBlockingReader()
	28	+	defer in.unblock()
	29	+	term.EXPECT().In().Return(in).Maybe()
	30	+	term.EXPECT().Out().Return(&out).Maybe()
	31	+	term.EXPECT().Flush().Run(func() { close(flushed) }).Return(nil).Once()
	32	+	term.EXPECT().ResizeEvents().Return(resizeCh).Maybe()
	33	+
	34	+	observer := &blockingClientRuntimeObserver{entered: make(chan struct{}), release: make(chan struct{})}
	35	+	acked := make(chan struct{})
	36	+	transport := portsmocks.NewMockTransport(t)
	37	+	transport.EXPECT().Send(isType(ports.MsgHello)).Return(nil).Once()
	38	+	transport.EXPECT().Send(isType(ports.MsgAck)).Run(func(ports.Frame) { close(acked) }).Return(nil).Once()
	39	+	unblock := scriptRecv(transport,
	40	+	recvItem{f: frameOf(ports.MsgWelcome, ports.MarshalWelcome(ports.Welcome{SessionID: "s1"}))},
	41	+	recvItem{f: frameOf(ports.MsgOutput, ports.MarshalOutput(ports.Output{Data: []byte("flush-before-observe"), NewStateNum: 3}))},
	42	+	)
	43	+	defer unblock()
	44	+	transport.EXPECT().Close().Return(nil).Once()
	45	+
	46	+	ctx, cancel := context.WithCancel(context.Background())
	47	+	result := make(chan error, 1)
	48	+	go func() {
	49	+	result <- client.Attach(ctx, transport, term, realClock{}, ports.IntentEphemeral, "", client.WithRuntimeObserver(observer))
	50	+	}()
	51	+	awaitClientRuntime(t, observer.entered, "blocking observer")
	52	+	awaitClientRuntime(t, flushed, "terminal flush")
	53	+	awaitClientRuntime(t, acked, "ACK")
	54	+	cancel()
	55	+	close(observer.release)
	56	+	select {
	57	+	case err := <-result:
	58	+	require.NoError(t, err)
	59	+	case <-time.After(time.Second):
	60	+	t.Fatal("Attach did not complete after observer release")
	61	+	}
	62	+	require.Equal(t, "flush-before-observe", out.String())
	63	+	require.Equal(t, int32(1), restores.Load())
	64	+	}
	65	+
	66	+	func TestTerminalFlushBoundaryTransportObservability(t *testing.T) {
	67	+	var out bytes.Buffer
	68	+	var restores atomic.Int32
	69	+	resizeCh := make(chan domain.Size)
	70	+	term, in := newHappyTerminal(t, &out, &restores, resizeCh)
	71	+	defer in.unblock()
	72	+
	73	+	observer := &clientRuntimeObserver{}
	74	+	transport := portsmocks.NewMockTransport(t)
	75	+	transport.EXPECT().Send(isType(ports.MsgHello)).Return(nil).Once()
	76	+	unblock := scriptRecv(transport,
	77	+	recvItem{f: frameOf(ports.MsgWelcome, ports.MarshalWelcome(ports.Welcome{SessionID: "s1"}))},
	78	+	recvItem{f: frameOf(ports.MsgOutput, ports.MarshalOutput(ports.Output{Data: []byte("unchanged-by-observer"), NewStateNum: 3}))},
	79	+	recvItem{f: frameOf(ports.MsgDetached, ports.MarshalDetached(ports.Detached{Reason: ports.ReasonDetach}))},
	80	+	)
	81	+	defer unblock()
	82	+	transport.EXPECT().Send(isType(ports.MsgAck)).Return(nil).Maybe()
	83	+	transport.EXPECT().Close().Return(nil).Once()
	84	+
	85	+	err := client.Attach(context.Background(), transport, term, realClock{}, ports.IntentEphemeral, "", client.WithRuntimeObserver(observer))
	86	+	require.NoError(t, err)
	87	+	require.Equal(t, "unchanged-by-observer", out.String(), "observer must not alter terminal bytes")
	88	+	require.Equal(t, int32(1), restores.Load())
	89	+	// Carriage spans belong only to concrete adapters. The client owns the
	90	+	// post-successful-flush terminal boundary, before its ACK is queued.
	91	+	observer.requireOrdered(t, ports.RuntimeTerminalFlushed)
	92	+	}
	93	+
	94	+	type blockingClientRuntimeObserver struct {
	95	+	entered chan struct{}
	96	+	release chan struct{}
	97	+	once sync.Once
	98	+	}
	99	+
	100	+	func (o *blockingClientRuntimeObserver) ObserveRuntime(ports.RuntimeMark) {
	101	+	o.once.Do(func() {
	102	+	close(o.entered)
	103	+	<-o.release
	104	+	})
	105	+	}
	106	+
	107	+	func awaitClientRuntime(t *testing.T, ch <-chan struct{}, what string) {
	108	+	t.Helper()
	109	+	select {
	110	+	case <-ch:
	111	+	case <-time.After(time.Second):
	112	+	t.Fatalf("timed out waiting for %s", what)
	113	+	}
	114	+	}
	115	+
	116	+	type clientRuntimeObserver struct {
	117	+	mu sync.Mutex
	118	+	marks []ports.RuntimeMark
	119	+	}
	120	+
	121	+	func (o *clientRuntimeObserver) ObserveRuntime(mark ports.RuntimeMark) {
	122	+	o.mu.Lock()
	123	+	defer o.mu.Unlock()
	124	+	o.marks = append(o.marks, mark)
	125	+	}
	126	+
	127	+	func (o clientRuntimeObserver) requireOrdered(t testing.T, kinds ...ports.RuntimeMarkKind) {
	128	+	t.Helper()
	129	+	o.mu.Lock()
	130	+	defer o.mu.Unlock()
	131	+	next := 0
	132	+	for _, mark := range o.marks {
	133	+	if next < len(kinds) && mark.Kind == kinds[next] {
	134	+	next++
	135	+	}
	136	+	}
	137	+	if next != len(kinds) {
	138	+	t.Fatalf("runtime marks %#v do not contain ordered %#v", o.marks, kinds)
	139	+	}
	140	+	}

M internal/usecase/daemon/client.go +2 -1


@@ -424,7 +424,8 @@ func (d Daemon) attachCoordinator(sess session, old, current *attachedClient,
424	424		rc := sess.renderCoordinator()
425	425		if rc == nil {
426	426		rc = newRenderCoordinator(renderCoordinatorOptions{
427		-	clock: d.clock,
	427	+	clock: d.clock,
	428	+	observer: d.runtimeObserver,
428	429		wake: func(w renderWake) {
429	430		// Never reread sess.client here: a wake is bound to both its
430	431		// attachment and its coordinator incarnation. The second check in

M internal/usecase/daemon/daemon.go +59 -21


@@ -22,8 +22,9 @@ import (
22	22		// Scheduler debounce bounds. Idle updates use the minimum for low latency;
23	23		// sustained floods step toward the maximum to reduce frame/syscall pressure.
24	24		const (
25		-	minDebounceInterval = 2 * time.Millisecond
26		-	maxSyncUpdateDuration = 500 * time.Millisecond
	25	+	minDebounceInterval = 2 * time.Millisecond
	26	+	maxSyncUpdateDuration = 500 * time.Millisecond
	27	+	runtimeObserverQueueDepth = 64
27	28		)
28	29
29	30		const debounceInterval = minDebounceInterval

@@ -92,25 +93,28 @@ type Daemon struct {
92	93		// interval between JRS validation and its committed hand-off.
93	94		beforeRecentSessionHandoff func()
94	95
95		-	ptys ports.PTYFactory
96		-	clock ports.Clock
97		-	log *slog.Logger
98		-	baseEnv []string
99		-	shell string
100		-	shellArgs []string
101		-	persist *persist.Persister
102		-	persistEnabled bool
103		-	snaps ports.SnapshotStore
104		-	snapsEnabled bool
105		-	snapshotMarshal func(snapcodec.Session) ([]byte, error)
106		-	snapshotJobs chan *snapshotCapture
107		-	snapshotWorkerMu sync.Mutex
108		-	snapshotWorkerID uint64
109		-	snapshotWorkerCtx context.Context
110		-	snapshotWorkerCancel context.CancelFunc
111		-	snapshotWorkerDone chan struct{}
112		-	snapshotWorkerFlush chan struct{}
113		-	snapshotWorkerFinalWake chan struct{}
	96	+	ptys ports.PTYFactory
	97	+	clock ports.Clock
	98	+	log *slog.Logger
	99	+	runtimeObserver ports.RuntimeObserver
	100	+	runtimeObserverCloser ports.SerializedRuntimeObserver
	101	+	runtimeObserverCloseOnce sync.Once
	102	+	baseEnv []string
	103	+	shell string
	104	+	shellArgs []string
	105	+	persist *persist.Persister
	106	+	persistEnabled bool
	107	+	snaps ports.SnapshotStore
	108	+	snapsEnabled bool
	109	+	snapshotMarshal func(snapcodec.Session) ([]byte, error)
	110	+	snapshotJobs chan *snapshotCapture
	111	+	snapshotWorkerMu sync.Mutex
	112	+	snapshotWorkerID uint64
	113	+	snapshotWorkerCtx context.Context
	114	+	snapshotWorkerCancel context.CancelFunc
	115	+	snapshotWorkerDone chan struct{}
	116	+	snapshotWorkerFlush chan struct{}
	117	+	snapshotWorkerFinalWake chan struct{}
114	118		// snapshotFinalJobs retains immutable terminal captures rejected by the
115	119		// bounded regular queue. It is only populated during session teardown and
116	120		// drained by the same worker, so persistence remains outside daemon locks.

@@ -183,6 +187,39 @@ func (s stoppedSession) same(other stoppedSession) bool {
183	187
184	188		type Option func(*Daemon)
185	189
	190	+	// WithRuntimeObserver enables opt-in process-local marks. It takes no clock:
	191	+	// the concrete observer is the sole timestamp owner.
	192	+	func WithRuntimeObserver(observer ports.RuntimeObserver) Option {
	193	+	return func(d *Daemon) {
	194	+	// Options are applied before the daemon starts. Replacing an owned
	195	+	// observer here cannot race producers and must retire its worker.
	196	+	if d.runtimeObserverCloser != nil {
	197	+	d.runtimeObserverCloser.Close()
	198	+	d.runtimeObserverCloser = nil
	199	+	}
	200	+	reporter, owned := ports.EnsureSerializedRuntimeObserver(observer, runtimeObserverQueueDepth)
	201	+	d.runtimeObserver = reporter
	202	+	if owned {
	203	+	d.runtimeObserverCloser = reporter
	204	+	}
	205	+	}
	206	+	}
	207	+
	208	+	// closeRuntimeObserver drains the daemon-owned reporter before teardown
	209	+	// completes. Application-owned process reporters are closed by their app
	210	+	// lifecycle instead, so shared observers never receive a duplicate close.
	211	+	func (d *Daemon) closeRuntimeObserver() {
	212	+	if d == nil {
	213	+	return
	214	+	}
	215	+	d.runtimeObserverCloseOnce.Do(func() {
	216	+	if d.runtimeObserverCloser != nil {
	217	+	d.runtimeObserverCloser.Flush()
	218	+	d.runtimeObserverCloser.Close()
	219	+	}
	220	+	})
	221	+	}
	222	+
186	223		// WithShell overrides the command (and its args) each session spawns. The
187	224		// default is $SHELL (or /bin/sh) with no arguments; tests use this to run a
188	225		// deterministic program.

@@ -356,6 +393,7 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt
356	393		// returns when the last session is removed or ctx is cancelled; on the latter
357	394		// path attached clients are detached with ReasonServerShutdown.
358	395		func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error {
	396	+	defer d.closeRuntimeObserver()
359	397		d.serveCtx, d.serveCancel = context.WithCancel(ctx)
360	398		defer d.serveCancel()
361	399		d.hardCtx, d.hardCancel = context.WithCancel(context.Background())

M internal/usecase/daemon/render.go +56 -2


@@ -5,6 +5,7 @@ import (
5	5		"bytes"
6	6
7	7		"github.com/bnema/vev/internal/domain"
	8	+	"github.com/bnema/vev/internal/ports"
8	9		"github.com/bnema/vev/internal/usecase/layout"
9	10		themeui "github.com/bnema/vev/internal/usecase/theme"
10	11		"github.com/bnema/vev/pkg/renderer"

@@ -193,7 +194,50 @@ func (d Daemon) flushPTYEffects(sess session, tb tab, p pane) {
193	194		// paint draws the composed client frame (active tab plus status bar) and
194	195		// sends the resulting bytes. The renderer shadow is reset on explicit invalidations
195	196		// such as switch/create/close/rename/resize so the repaint is complete.
	197	+	func (d *Daemon) observeRuntime(kind ports.RuntimeMarkKind, bytes uint64, valid bool) {
	198	+	if d != nil && d.runtimeObserver != nil {
	199	+	d.runtimeObserver.ObserveRuntime(ports.NewRuntimeMark("daemon", kind, bytes, valid))
	200	+	}
	201	+	}
	202	+
	203	+	// runtimeMarkBatch captures immutable runtime marks during a render transaction.
	204	+	// paint owns attachment sendMu while it captures, composes, and emits; JSONL
	205	+	// observer I/O must therefore happen only after that ownership is released.
	206	+	type runtimeMarkBatch struct {
	207	+	observer ports.RuntimeObserver
	208	+	marks []ports.RuntimeMark
	209	+	}
	210	+
	211	+	func (d *Daemon) newRuntimeMarkBatch() runtimeMarkBatch {
	212	+	if d == nil {
	213	+	return runtimeMarkBatch{}
	214	+	}
	215	+	return runtimeMarkBatch{observer: d.runtimeObserver}
	216	+	}
	217	+
	218	+	func (b *runtimeMarkBatch) span(start, end ports.RuntimeMarkKind, bytes uint64) func(uint64, bool) {
	219	+	if b == nil \|\| b.observer == nil {
	220	+	return func(uint64, bool) {}
	221	+	}
	222	+	correlation := ports.NewRuntimeCorrelation()
	223	+	b.marks = append(b.marks, ports.NewRuntimeMarkWithCorrelation("daemon", correlation, start, bytes, true))
	224	+	return func(endBytes uint64, valid bool) {
	225	+	b.marks = append(b.marks, ports.NewRuntimeMarkWithCorrelation("daemon", correlation, end, endBytes, valid))
	226	+	}
	227	+	}
	228	+
	229	+	func (b *runtimeMarkBatch) flush() {
	230	+	if b == nil \|\| b.observer == nil {
	231	+	return
	232	+	}
	233	+	for _, mark := range b.marks {
	234	+	b.observer.ObserveRuntime(mark)
	235	+	}
	236	+	}
	237	+
196	238		func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...uint64) {
	239	+	marks := d.newRuntimeMarkBatch()
	240	+
197	241		attachmentEpoch := uint64(0)
198	242		if len(epochs) != 0 {
199	243		attachmentEpoch = epochs[0]

@@ -218,6 +262,8 @@ func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...
218	262		// the normal gate, but this ownership check also protects direct resize and
219	263		// test paint paths from consuming damage that cannot be emitted.
220	264		if ac.output != nil && ac.output.atCapacity() {
	265	+	// The coordinator owns the blocked interval; this guard only protects
	266	+	// direct test/resize paint calls from destructive capture.
221	267		ac.sendMu.Unlock()
222	268		return
223	269		}

@@ -228,11 +274,15 @@ func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...
228	274		overlays := ac.overlays.SnapshotForRender()
229	275		repaintAttachedClients := false
230	276		defer func() {
	277	+	overlays.Unlock()
	278	+	// Every return below has released sendMu before reaching this defer.
	279	+	// Emit the captured sequence before any follow-up repaint can introduce
	280	+	// another boundary, while no attachment, session, tab, or pane lock is held.
	281	+	marks.flush()
231	282		if repaintAttachedClients {
232	283		d.repaintAllAttachedClients()
233	284		}
234	285		}()
235		-	defer overlays.Unlock()
236	286		preview := snapshotPickerPreview(nil)
237	287		if overlays.previewTab != tb {
238	288		preview = snapshotPickerPreview(overlays.previewTab)

@@ -270,7 +320,9 @@ func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...
270	320		pickerActive: overlays.pickerActive, paletteActive: overlays.paletteActive, promptActive: overlays.promptActive,
271	321		copyFeedback: overlays.copyFeedback,
272	322		}
	323	+	endCapture := marks.span(ports.RuntimeCaptureStart, ports.RuntimeCaptureEnd, 0)
273	324		state, ok := captureRenderState(sess, ac, bars, capturedOverlays, preview, floatingCfg, reset, damageCaptureConsume)
	325	+	endCapture(0, ok)
274	326		if !ok {
275	327		ac.sendMu.Unlock()
276	328		return

@@ -285,11 +337,13 @@ func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...
285	337		state.preview = pickerPreviewFromCapturedRender(previewState)
286	338		}
287	339		captureOverlayLayers(state, overlays, paletteCfg)
	340	+	endCompose := marks.span(ports.RuntimeComposeStart, ports.RuntimeComposeEnd, 0)
288	341		composed := composeFrame(*state, ac.pipelineCache, ac.pipelineScratch)
	342	+	endCompose(0, true)
289	343		if ac.renderStages.compose != nil {
290	344		ac.renderStages.compose()
291	345		}
292		-	d.emitFrame(sess, ac, state, composed)
	346	+	d.emitFrame(sess, ac, state, composed, &marks)
293	347		}
294	348
295	349		type themeStyles struct {

M internal/usecase/daemon/render_coordinator.go +131 -1


@@ -78,7 +78,8 @@ type resizeRequestMetadata struct {
78	78
79	79		// renderCoordinatorOptions wires one coordinator instance.
80	80		type renderCoordinatorOptions struct {
81		-	clock ports.Clock
	81	+	clock ports.Clock
	82	+	observer ports.RuntimeObserver
82	83		// wake is the transitional composition target.
83	84		wake func(renderWake)
84	85		// ackReady reports whether the attachment may compose another output

@@ -107,6 +108,56 @@ type renderCoordinatorBurstMetrics struct {
107	108		coalesced atomic.Uint64
108	109		}
109	110
	111	+	// ackBlockedSpan hands an ACK-blocked endpoint pair between coordinator
	112	+	// transitions without holding c.mu while the observer runs. A consuming ACK
	113	+	// may arrive while Start is blocked in observer I/O; finish records that
	114	+	// closure and publishStart emits it only after Start has returned.
	115	+	type ackBlockedSpan struct {
	116	+	observer ports.RuntimeObserver
	117	+	correlation ports.RuntimeCorrelation
	118	+
	119	+	mu sync.Mutex
	120	+	startPublished bool
	121	+	finished bool
	122	+	endPending bool
	123	+	endValid bool
	124	+	}
	125	+
	126	+	func newACKBlockedSpan(observer ports.RuntimeObserver) *ackBlockedSpan {
	127	+	return &ackBlockedSpan{observer: observer, correlation: ports.NewRuntimeCorrelation()}
	128	+	}
	129	+
	130	+	func (s *ackBlockedSpan) publishStart() {
	131	+	s.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", s.correlation, ports.RuntimeACKBlockedStart, 0, true))
	132	+
	133	+	s.mu.Lock()
	134	+	s.startPublished = true
	135	+	endPending, endValid := s.endPending, s.endValid
	136	+	s.endPending = false
	137	+	s.mu.Unlock()
	138	+	if endPending {
	139	+	s.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", s.correlation, ports.RuntimeACKBlockedEnd, 0, endValid))
	140	+	}
	141	+	}
	142	+
	143	+	// finish closes a span at most once. It never invokes the observer before the
	144	+	// blocked start has completed publication.
	145	+	func (s *ackBlockedSpan) finish(valid bool) {
	146	+	s.mu.Lock()
	147	+	if s.finished {
	148	+	s.mu.Unlock()
	149	+	return
	150	+	}
	151	+	s.finished = true
	152	+	if !s.startPublished {
	153	+	s.endPending, s.endValid = true, valid
	154	+	s.mu.Unlock()
	155	+	return
	156	+	}
	157	+	s.mu.Unlock()
	158	+	s.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", s.correlation, ports.RuntimeACKBlockedEnd, 0, valid))
	159	+	}
	160	+
110	161		// renderCoordinatorBurstMetricsSnapshot is an immutable, internal view for
111	162		// benchmark reporting. Scheduling only writes the atomic counters above.
112	163		type renderCoordinatorBurstMetricsSnapshot struct {

@@ -124,6 +175,10 @@ type renderCoordinator struct {
124	175		pending bool
125	176		pendingReset bool
126	177		pendingUrgent bool
	178	+	// queueMarked spans the coordinator's actual pending interval, from first
	179	+	// enqueue until fire consumes the coalesced work.
	180	+	queueMarked bool
	181	+	queueCorrelation ports.RuntimeCorrelation
127	182		// pendingPreview is accounted separately from the target primary wake:
128	183		// viewers may consume a coalesced target snapshot while that target is
129	184		// still ACK-blocked.

@@ -131,6 +186,9 @@ type renderCoordinator struct {
131	186		// ackDeferred records that an expired deadline or watchdog was blocked by
132	187		// output-window capacity. ACK notifications may flush only this state.
133	188		ackDeferred bool
	189	+	// ackBlocked retains the first rejected ACK-capacity probe until the
	190	+	// matching consume or lifecycle closure releases that exact interval.
	191	+	ackBlocked *ackBlockedSpan
134	192		// deadlineDue closes the handoff between a timer worker receiving its tick
135	193		// and fire observing unavailable ACK capacity.
136	194		deadlineDue bool

@@ -287,9 +345,17 @@ func (c renderCoordinator) invalidateForAttachmentAtResizeEpoch(source attache
287	345		onInvalidate := c.opts.onInvalidate
288	346		c.metrics.invalidations.Add(1)
289	347		wasPending, wasUrgent, wasPreviewPending := c.pending, c.pendingUrgent, c.pendingPreview
	348	+	queueStart := false
	349	+	var queueCorrelation ports.RuntimeCorrelation
290	350		if !wasPending {
291	351		c.ackDeferred = false
292	352		c.deadlineDue = false
	353	+	if c.opts.observer != nil {
	354	+	c.queueMarked = true
	355	+	c.queueCorrelation = ports.NewRuntimeCorrelation()
	356	+	queueCorrelation = c.queueCorrelation
	357	+	queueStart = true
	358	+	}
293	359		}
294	360		c.pending = true
295	361		c.pendingReset = c.pendingReset \|\| inv.reset

@@ -312,7 +378,11 @@ func (c renderCoordinator) invalidateForAttachmentAtResizeEpoch(source attache
312	378		}
313	379		clock := c.opts.clock
314	380		c.armed = c.armed \|\| arm
	381	+	observer := c.opts.observer
315	382		c.mu.Unlock()
	383	+	if queueStart && observer != nil {
	384	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueEnqueued, 0, true))
	385	+	}
316	386		stopTimer(old)
317	387		if !arm \|\| clock == nil {
318	388		if onInvalidate != nil {

@@ -377,7 +447,12 @@ func (c *renderCoordinator) notifyAck() {
377	447		return
378	448		}
379	449		gen := c.generation
	450	+	blocked := c.ackBlocked
	451	+	c.ackBlocked = nil
380	452		c.mu.Unlock()
	453	+	if blocked != nil {
	454	+	blocked.finish(true)
	455	+	}
381	456		c.fire(gen, false, false)
382	457		}
383	458

@@ -635,6 +710,10 @@ func (c *renderCoordinator) wakeCurrent(w renderWake) bool {
635	710		func (c renderCoordinator) noteDetach(ac attachedClient) {
636	711		c.mu.Lock()
637	712		var timer, resizeTimer, retryTimer ports.Timer
	713	+	var observer ports.RuntimeObserver
	714	+	var queueCorrelation ports.RuntimeCorrelation
	715	+	var queueMarked bool
	716	+	var ackBlocked *ackBlockedSpan
638	717		if c.attachment == ac {
639	718		c.advanceAttachmentEpochLocked(ac)
640	719		c.attachment = nil

@@ -660,11 +739,20 @@ func (c renderCoordinator) noteDetach(ac attachedClient) {
660	739		}
661	740		retryTimer, c.retryTimer = c.retryTimer, nil
662	741		c.retryGen++
	742	+	observer, queueCorrelation, queueMarked = c.opts.observer, c.queueCorrelation, c.queueMarked
	743	+	ackBlocked = c.ackBlocked
	744	+	c.queueMarked, c.ackBlocked = false, nil
663	745		}
664	746		c.mu.Unlock()
665	747		stopTimer(timer)
666	748		stopTimer(resizeTimer)
667	749		stopTimer(retryTimer)
	750	+	if queueMarked && observer != nil {
	751	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueDequeued, 0, false))
	752	+	}
	753	+	if ackBlocked != nil {
	754	+	ackBlocked.finish(false)
	755	+	}
668	756		}
669	757
670	758		// noteReplace hands the coordinator from old to replacement; callbacks

@@ -675,6 +763,10 @@ func (c renderCoordinator) noteReplace(old, replacement attachedClient, readin
675	763		ready = readiness[0]
676	764		}
677	765		c.mu.Lock()
	766	+	var observer ports.RuntimeObserver
	767	+	var queueCorrelation ports.RuntimeCorrelation
	768	+	var queueMarked bool
	769	+	var ackBlocked *ackBlockedSpan
678	770		// A coordinator may be installed while replacing a legacy attachment
679	771		// which predated coordinator ownership. In that case nil has no pending
680	772		// identity to invalidate, and the replacement becomes the first bound

@@ -708,11 +800,20 @@ func (c renderCoordinator) noteReplace(old, replacement attachedClient, readin
708	800		}
709	801		retryTimer, c.retryTimer = c.retryTimer, nil
710	802		c.retryGen++
	803	+	observer, queueCorrelation, queueMarked = c.opts.observer, c.queueCorrelation, c.queueMarked
	804	+	ackBlocked = c.ackBlocked
	805	+	c.queueMarked, c.ackBlocked = false, nil
711	806		}
712	807		c.mu.Unlock()
713	808		stopTimer(timer)
714	809		stopTimer(resizeTimer)
715	810		stopTimer(retryTimer)
	811	+	if queueMarked && observer != nil {
	812	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueDequeued, 0, false))
	813	+	}
	814	+	if ackBlocked != nil {
	815	+	ackBlocked.finish(false)
	816	+	}
716	817		}
717	818
718	819		// notePark invalidates pending wakes when the attachment parks for resume.

@@ -747,7 +848,16 @@ func (c *renderCoordinator) noteSessionTeardown() {
747	848		c.armed = false
748	849		timer := c.detachNormalTimerLocked()
749	850		timers := c.detachSyncTimersLocked()
	851	+	observer, queueCorrelation, queueMarked := c.opts.observer, c.queueCorrelation, c.queueMarked
	852	+	ackBlocked := c.ackBlocked
	853	+	c.queueMarked, c.ackBlocked = false, nil
750	854		c.mu.Unlock()
	855	+	if queueMarked && observer != nil {
	856	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueDequeued, 0, false))
	857	+	}
	858	+	if ackBlocked != nil {
	859	+	ackBlocked.finish(false)
	860	+	}
751	861		stopTimer(timer)
752	862		stopTimer(resizeTimer)
753	863		stopTimer(retryTimer)

@@ -1052,11 +1162,22 @@ func (c *renderCoordinator) fire(gen uint64, watchdog, deadline bool) {
1052	1162		// notification rather than accumulating ackDeferred forever.
1053	1163		headlessPreviewOnly := c.detached && c.attachment == nil
1054	1164		if !headlessPreviewOnly && (!ready \|\| (c.attachment != nil && !c.attachmentReady)) {
	1165	+	var ackStart *ackBlockedSpan
1055	1166		if !ready && deadline {
1056	1167		c.ackDeferred = true
	1168	+	if c.ackBlocked == nil && c.opts.observer != nil {
	1169	+	// Install the span while coordinator state is owned. Its Start
	1170	+	// publication runs unlocked, and its own handoff defers End until
	1171	+	// that call has returned.
	1172	+	ackStart = newACKBlockedSpan(c.opts.observer)
	1173	+	c.ackBlocked = ackStart
	1174	+	}
1057	1175		}
1058	1176		preview, previews := c.takePendingPreviewsLocked()
1059	1177		c.mu.Unlock()
	1178	+	if ackStart != nil {
	1179	+	ackStart.publishStart()
	1180	+	}
1060	1181		c.notifyPreviews(w, preview, previews)
1061	1182		return
1062	1183		}

@@ -1075,9 +1196,18 @@ func (c *renderCoordinator) fire(gen uint64, watchdog, deadline bool) {
1075	1196		}
1076	1197		timer := c.detachNormalTimerLocked()
1077	1198		c.pending, c.pendingReset, c.pendingUrgent, c.ackDeferred, c.deadlineDue, c.coalesced, c.armed = false, false, false, false, false, 0, false
	1199	+	observer, queueCorrelation, queueMarked := c.opts.observer, c.queueCorrelation, c.queueMarked
	1200	+	ackBlocked := c.ackBlocked
	1201	+	c.queueMarked, c.ackBlocked = false, nil
1078	1202		wake := c.opts.wake
1079	1203		c.mu.Unlock()
1080	1204		stopTimer(timer)
	1205	+	if queueMarked && observer != nil {
	1206	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueDequeued, 0, true))
	1207	+	}
	1208	+	if ackBlocked != nil {
	1209	+	ackBlocked.finish(true)
	1210	+	}
1081	1211		if wake != nil && !headlessPreviewOnly {
1082	1212		wake(w)
1083	1213		}

M internal/usecase/daemon/render_pipeline.go +16 -1


@@ -286,7 +286,18 @@ func commitDamageReceipts(receipts []damageReceipt) {
286	286
287	287		// emitFrame is the sole side-effecting half of the pipeline. The caller holds
288	288		// sendMu for the complete capture/compose/emit transaction.
289		-	func (d Daemon) emitFrame(sess session, ac attachedClient, state capturedRenderState, composed composedRenderFrame) bool {
	289	+	func (d Daemon) emitFrame(sess session, ac attachedClient, state capturedRenderState, composed composedRenderFrame, batches ...*runtimeMarkBatch) bool {
	290	+	var ownedMarks runtimeMarkBatch
	291	+	var marks *runtimeMarkBatch
	292	+	if len(batches) != 0 {
	293	+	marks = batches[0]
	294	+	} else {
	295	+	ownedMarks = d.newRuntimeMarkBatch()
	296	+	marks = &ownedMarks
	297	+	// Direct test callers retain the same guarantee as paint: observer I/O
	298	+	// follows emitFrame's release of attachment ownership.
	299	+	defer marks.flush()
	300	+	}
290	301		if sess == nil \|\| ac == nil \|\| state == nil {
291	302		if ac != nil {
292	303		ac.sendMu.Unlock()

@@ -306,7 +317,9 @@ func (d Daemon) emitFrame(sess session, ac attachedClient, state capturedRen
306	317		ac.sendMu.Unlock()
307	318		return false
308	319		}
	320	+	endDiff := marks.span(ports.RuntimeDiffStart, ports.RuntimeDiffEnd, 0)
309	321		prepared, err := ac.output.prepare(composed.frame, composed.damage, composed.reset)
	322	+	endDiff(0, err == nil)
310	323		if err != nil {
311	324		ac.sess.mu.Unlock()
312	325		ac.sendMu.Unlock()

@@ -319,6 +332,7 @@ func (d Daemon) emitFrame(sess session, ac attachedClient, state capturedRen
319	332		var sendErr error
320	333		if len(data) > 0 {
321	334		sendTr = ac.transport()
	335	+	endEmit := marks.span(ports.RuntimeEmitStart, ports.RuntimeEmitEnd, uint64(len(data)))
322	336		if sendTr == nil {
323	337		sendErr = errors.New("client transport is nil")
324	338		} else {

@@ -328,6 +342,7 @@ func (d Daemon) emitFrame(sess session, ac attachedClient, state capturedRen
328	342		}
329	343		sendErr = prepared.send(data, ac.echoAck.Load(), send)
330	344		}
	345	+	endEmit(uint64(len(data)), sendErr == nil)
331	346		}
332	347		if sendErr == nil {
333	348		// Publish only after output preparation and transport emission both

M internal/usecase/daemon/transactional_resize.go +5 -1


@@ -2,6 +2,7 @@ package daemon
2	2
3	3		import (
4	4		"github.com/bnema/vev/internal/domain"
	5	+	"github.com/bnema/vev/internal/ports"
5	6		"github.com/bnema/vev/internal/usecase/layout"
6	7		)
7	8

@@ -161,6 +162,7 @@ func (d Daemon) commitResize(sess session, ac *attachedClient, plan resizePlan
161	162		ac.sendMu.Unlock()
162	163		}
163	164		markSnapshotDirty(sess)
	165	+	d.observeRuntime(ports.RuntimeResizeCommitted, 0, true)
164	166		}
165	167
166	168		func (d Daemon) runResizeTransaction(sess session, ac *attachedClient, epoch uint64) bool {

@@ -249,7 +251,9 @@ func (d Daemon) retryResizeMembers(sess session, ac *attachedClient, epoch uin
249	251		// attached requests, and coordinator schedule acceptance for async requests.
250	252		// Headless requests have no reset invalidation and report geometry completion.
251	253		func (d Daemon) requestTransactionalResize(sess session, ac *attachedClient, size domain.Size, immediate bool) bool {
252		-	if sess == nil \|\| !size.Valid() {
	254	+	valid := sess != nil && size.Valid()
	255	+	d.observeRuntime(ports.RuntimeResizeRequested, 0, valid)
	256	+	if !valid {
253	257		return false
254	258		}
255	259		if ac == nil {

A internal/usecase/daemon/transport_observability_test.go +388 -0

@@ -0,0 +1,388 @@
	1	+	package daemon
	2	+
	3	+	import (
	4	+	"runtime"
	5	+	"sync"
	6	+	"sync/atomic"
	7	+	"testing"
	8	+	"time"
	9	+
	10	+	"github.com/bnema/vev/internal/domain"
	11	+	"github.com/bnema/vev/internal/ports"
	12	+	)
	13	+
	14	+	// This acceptance boundary exercises the attached resize, render, and ACK
	15	+	// path rather than merely listing the marks the implementation should emit.
	16	+	func TestTransportObservabilityDaemonBoundaries(t *testing.T) {
	17	+	observer := &daemonRuntimeObserver{}
	18	+	d, sess, ac, sends := newManualSessionWithPTYs(t, &quietPTY{})
	19	+	d.runtimeObserver = observer
	20	+	// One in-flight state makes the second public resize take the ACK-blocked
	21	+	// path. The real coordinator owns both renders and the ACK wake.
	22	+	ac.output = newOutputStateStream(1)
	23	+	rc := d.attachCoordinator(sess, nil, ac, true)
	24	+
	25	+	if !d.resizeForFirstPaint(sess, ac, domain.Size{Cols: 90, Rows: 25}) {
	26	+	t.Fatal("first resize was not accepted")
	27	+	}
	28	+	first := <-sends
	29	+	if first.Type != ports.MsgOutput {
	30	+	t.Fatalf("first frame = %v, want output", first.Type)
	31	+	}
	32	+	output, err := ports.UnmarshalOutput(first.Payload)
	33	+	if err != nil {
	34	+	t.Fatalf("decode first output: %v", err)
	35	+	}
	36	+	if !d.resizeForFirstPaint(sess, ac, domain.Size{Cols: 100, Rows: 26}) {
	37	+	t.Fatal("second resize was not accepted")
	38	+	}
	39	+	ac.ackOutputState(output.NewStateNum)
	40	+	rc.notifyAck()
	41	+	second := <-sends
	42	+	if second.Type != ports.MsgOutput {
	43	+	t.Fatalf("second frame = %v, want output", second.Type)
	44	+	}
	45	+
	46	+	want := []ports.RuntimeMarkKind{
	47	+	ports.RuntimeResizeRequested, ports.RuntimeResizeCommitted,
	48	+	ports.RuntimeQueueEnqueued, ports.RuntimeQueueDequeued,
	49	+	ports.RuntimeCaptureStart, ports.RuntimeCaptureEnd,
	50	+	ports.RuntimeComposeStart, ports.RuntimeComposeEnd,
	51	+	ports.RuntimeDiffStart, ports.RuntimeDiffEnd,
	52	+	ports.RuntimeEmitStart, ports.RuntimeEmitEnd,
	53	+	ports.RuntimeResizeRequested, ports.RuntimeResizeCommitted,
	54	+	ports.RuntimeQueueEnqueued, ports.RuntimeACKBlockedStart,
	55	+	ports.RuntimeACKBlockedEnd, ports.RuntimeQueueDequeued,
	56	+	ports.RuntimeCaptureStart, ports.RuntimeCaptureEnd,
	57	+	ports.RuntimeComposeStart, ports.RuntimeComposeEnd,
	58	+	ports.RuntimeDiffStart, ports.RuntimeDiffEnd,
	59	+	ports.RuntimeEmitStart, ports.RuntimeEmitEnd,
	60	+	}
	61	+	if len(observer.marks) != len(want) {
	62	+	t.Fatalf("runtime marks = %#v, want kinds %#v", observer.marks, want)
	63	+	}
	64	+	for i, kind := range want {
	65	+	if observer.marks[i].Kind != kind {
	66	+	t.Fatalf("runtime mark %d = %q, want %q; all=%#v", i, observer.marks[i].Kind, kind, observer.marks)
	67	+	}
	68	+	}
	69	+	}
	70	+
	71	+	type daemonRuntimeObserver struct{ marks []ports.RuntimeMark }
	72	+
	73	+	func (o *daemonRuntimeObserver) ObserveRuntime(mark ports.RuntimeMark) {
	74	+	o.marks = append(o.marks, mark)
	75	+	}
	76	+
	77	+	type blockingDaemonRuntimeObserver struct {
	78	+	mu sync.Mutex
	79	+	entered chan struct{}
	80	+	release chan struct{}
	81	+	blocked bool
	82	+	}
	83	+
	84	+	func newBlockingDaemonRuntimeObserver() *blockingDaemonRuntimeObserver {
	85	+	return &blockingDaemonRuntimeObserver{entered: make(chan struct{}), release: make(chan struct{})}
	86	+	}
	87	+
	88	+	func (o *blockingDaemonRuntimeObserver) ObserveRuntime(ports.RuntimeMark) {
	89	+	o.mu.Lock()
	90	+	defer o.mu.Unlock()
	91	+	if !o.blocked {
	92	+	o.blocked = true
	93	+	close(o.entered)
	94	+	<-o.release
	95	+	}
	96	+	}
	97	+
	98	+	func awaitDaemonObserver(t *testing.T, ch <-chan struct{}, what string) {
	99	+	t.Helper()
	100	+	select {
	101	+	case <-ch:
	102	+	case <-time.After(time.Second):
	103	+	t.Fatalf("timed out waiting for %s", what)
	104	+	}
	105	+	}
	106	+
	107	+	// A runtime observer may synchronously write JSONL. Its I/O must run only
	108	+	// after the render transaction has released attachment, session, and pane
	109	+	// ownership, so a blocked trace cannot stall live output processing.
	110	+	func TestTransportObservabilityBlockedRenderObserverReleasesArchitectureLocks(t *testing.T) {
	111	+	d, sess, ac, sends := newManualSessionWithPTYs(t, nil)
	112	+	observer := newBlockingDaemonRuntimeObserver()
	113	+	d.runtimeObserver = observer
	114	+
	115	+	paintDone := make(chan struct{})
	116	+	go func() {
	117	+	d.paint(sess, ac, true)
	118	+	close(paintDone)
	119	+	}()
	120	+	awaitDaemonObserver(t, observer.entered, "blocked render observer")
	121	+	frame := <-sends // The frame was emitted before the observer began its I/O.
	122	+	output, err := ports.UnmarshalOutput(frame.Payload)
	123	+	if err != nil {
	124	+	t.Fatalf("decode emitted output: %v", err)
	125	+	}
	126	+	ac.ackOutputState(output.NewStateNum)
	127	+
	128	+	pane := sess.activeTab().focusedPane()
	129	+	for _, lock := range []struct {
	130	+	name string
	131	+	fn func()
	132	+	}{
	133	+	{"attachment", func() { ac.sendMu.Lock(); runtime.Gosched(); ac.sendMu.Unlock() }},
	134	+	{"session", func() { sess.mu.Lock(); runtime.Gosched(); sess.mu.Unlock() }},
	135	+	{"pane", func() { pane.mu.Lock(); runtime.Gosched(); pane.mu.Unlock() }},
	136	+	} {
	137	+	done := make(chan struct{})
	138	+	go func(fn func()) { fn(); close(done) }(lock.fn)
	139	+	awaitDaemonObserver(t, done, lock.name+" lock")
	140	+	}
	141	+
	142	+	livePTY := make(chan struct{})
	143	+	go func() {
	144	+	d.processPTYData(sess, sess.activeTab(), pane, []byte("live"), false)
	145	+	close(livePTY)
	146	+	}()
	147	+	awaitDaemonObserver(t, livePTY, "live PTY processing")
	148	+
	149	+	// A second render can complete its actual pipeline while the first trace
	150	+	// write is blocked; it is not serialized behind the first attachment lock.
	151	+	secondPaint := make(chan struct{})
	152	+	go func() {
	153	+	d.paint(sess, ac, false)
	154	+	close(secondPaint)
	155	+	}()
	156	+	<-sends // The second frame is live even though serialized observer I/O waits.
	157	+
	158	+	close(observer.release)
	159	+	awaitDaemonObserver(t, paintDone, "first render completion")
	160	+	awaitDaemonObserver(t, secondPaint, "unrelated render completion")
	161	+	}
	162	+
	163	+	// ACK-blocked start is captured while coordinator state changes, then emitted
	164	+	// after c.mu is unlocked. This closes the remaining coordinator-observer path.
	165	+	func TestTransportObservabilityACKBlockedSpansEndExactlyOnce(t *testing.T) {
	166	+	cases := []struct {
	167	+	name string
	168	+	finish func(renderCoordinator, bool)
	169	+	}{
	170	+	{
	171	+	name: "notify ACK",
	172	+	finish: func(rc renderCoordinator, ready bool) {
	173	+	*ready = true
	174	+	rc.notifyAck()
	175	+	},
	176	+	},
	177	+	{
	178	+	name: "successful consume",
	179	+	finish: func(rc renderCoordinator, ready bool) {
	180	+	*ready = true
	181	+	rc.fire(rc.generation, false, false)
	182	+	},
	183	+	},
	184	+	{
	185	+	name: "teardown",
	186	+	finish: func(rc renderCoordinator, ready bool) {
	187	+	rc.noteSessionTeardown()
	188	+	},
	189	+	},
	190	+	}
	191	+	for _, tt := range cases {
	192	+	t.Run(tt.name, func(t *testing.T) {
	193	+	observer := &daemonRuntimeObserver{}
	194	+	ready := false
	195	+	rc := newRenderCoordinator(renderCoordinatorOptions{
	196	+	observer: observer,
	197	+	ackReady: func() bool { return ready },
	198	+	})
	199	+	rc.attach(&attachedClient{})
	200	+	rc.invalidate(renderInvalidation{class: invalidateOutput})
	201	+	rc.fire(rc.generation, false, true)
	202	+	tt.finish(rc, &ready)
	203	+	// A late notification after consuming or tearing down must not emit a
	204	+	// second end for the already-cleared blocked interval.
	205	+	rc.notifyAck()
	206	+
	207	+	var starts, ends []ports.RuntimeMark
	208	+	for _, mark := range observer.marks {
	209	+	switch mark.Kind {
	210	+	case ports.RuntimeACKBlockedStart:
	211	+	starts = append(starts, mark)
	212	+	case ports.RuntimeACKBlockedEnd:
	213	+	ends = append(ends, mark)
	214	+	}
	215	+	}
	216	+	if len(starts) != 1 \|\| len(ends) != 1 {
	217	+	t.Fatalf("ACK-blocked span marks: starts=%+v ends=%+v all=%+v", starts, ends, observer.marks)
	218	+	}
	219	+	if starts[0].Sequence != ends[0].Sequence \|\| starts[0].RequestID != ends[0].RequestID \|\| starts[0].Epoch != ends[0].Epoch {
	220	+	t.Fatalf("ACK-blocked span correlation changed: start=%+v end=%+v", starts[0], ends[0])
	221	+	}
	222	+	})
	223	+	}
	224	+	}
	225	+
	226	+	// A raw observer supplied through daemon configuration is serialized by the
	227	+	// daemon. Once Start has been published, a blocked End must not delay the ACK
	228	+	// that consumes deferred work and delivers its wake.
	229	+	func TestTransportObservabilityBlockedACKEndDoesNotDelayNotifyAck(t *testing.T) {
	230	+	observer := &blockingACKEndObserver{
	231	+	startObserved: make(chan struct{}),
	232	+	endEntered: make(chan struct{}),
	233	+	releaseEnd: make(chan struct{}),
	234	+	}
	235	+	d := New(nil, nil, nil, WithRuntimeObserver(observer))
	236	+	wakes := make(chan renderWake, 1)
	237	+	var ready atomic.Bool
	238	+	rc := newRenderCoordinator(renderCoordinatorOptions{
	239	+	observer: d.runtimeObserver,
	240	+	ackReady: ready.Load,
	241	+	wake: func(w renderWake) { wakes <- w },
	242	+	})
	243	+	rc.attach(&attachedClient{})
	244	+	rc.invalidate(renderInvalidation{class: invalidateOutput})
	245	+
	246	+	// Enter the ACK-blocked state and wait until Start has reached the injected
	247	+	// observer. End is the only deliberately blocked call in this test.
	248	+	rc.fire(rc.generation, false, true)
	249	+	awaitDaemonObserver(t, observer.startObserved, "ACK-blocked start observer")
	250	+
	251	+	ready.Store(true)
	252	+	ackDone := make(chan struct{})
	253	+	go func() {
	254	+	rc.notifyAck()
	255	+	close(ackDone)
	256	+	}()
	257	+	awaitDaemonObserver(t, observer.endEntered, "blocked ACK-blocked end observer")
	258	+	awaitDaemonObserver(t, ackDone, "nonblocking ACK progress")
	259	+	awaitWake(t, wakes)
	260	+	rc.mu.Lock()
	261	+	pending := rc.pending
	262	+	rc.mu.Unlock()
	263	+	if pending {
	264	+	t.Fatal("notifyAck left consumed ACK-deferred work pending")
	265	+	}
	266	+
	267	+	close(observer.releaseEnd)
	268	+	d.closeRuntimeObserver()
	269	+	}
	270	+
	271	+	type blockingACKEndObserver struct {
	272	+	startObserved chan struct{}
	273	+	endEntered chan struct{}
	274	+	releaseEnd chan struct{}
	275	+	}
	276	+
	277	+	func (o *blockingACKEndObserver) ObserveRuntime(mark ports.RuntimeMark) {
	278	+	switch mark.Kind {
	279	+	case ports.RuntimeACKBlockedStart:
	280	+	close(o.startObserved)
	281	+	case ports.RuntimeACKBlockedEnd:
	282	+	close(o.endEntered)
	283	+	<-o.releaseEnd
	284	+	}
	285	+	}
	286	+
	287	+	func TestTransportObservabilityBlockedCoordinatorObserverReleasesLock(t *testing.T) {
	288	+	observer := newBlockingDaemonRuntimeObserver()
	289	+	rc := newRenderCoordinator(renderCoordinatorOptions{
	290	+	observer: observer,
	291	+	ackReady: func() bool { return false },
	292	+	})
	293	+	rc.attach(&attachedClient{})
	294	+	rc.mu.Lock()
	295	+	rc.pending = true
	296	+	gen := rc.generation
	297	+	rc.mu.Unlock()
	298	+
	299	+	fired := make(chan struct{})
	300	+	go func() {
	301	+	rc.fire(gen, false, true)
	302	+	close(fired)
	303	+	}()
	304	+	awaitDaemonObserver(t, observer.entered, "blocked ACK observer")
	305	+
	306	+	acquired := make(chan struct{})
	307	+	go func() {
	308	+	rc.mu.Lock()
	309	+	runtime.Gosched()
	310	+	rc.mu.Unlock()
	311	+	close(acquired)
	312	+	}()
	313	+	awaitDaemonObserver(t, acquired, "coordinator lock")
	314	+
	315	+	close(observer.release)
	316	+	awaitDaemonObserver(t, fired, "ACK fire completion")
	317	+	}
	318	+
	319	+	type ackSpanInterleavingObserver struct {
	320	+	mu sync.Mutex
	321	+	marks []ports.RuntimeMark
	322	+	startEntered chan struct{}
	323	+	releaseStart chan struct{}
	324	+	}
	325	+
	326	+	func newACKSpanInterleavingObserver() *ackSpanInterleavingObserver {
	327	+	return &ackSpanInterleavingObserver{startEntered: make(chan struct{}), releaseStart: make(chan struct{})}
	328	+	}
	329	+
	330	+	func (o *ackSpanInterleavingObserver) ObserveRuntime(mark ports.RuntimeMark) {
	331	+	if mark.Kind == ports.RuntimeACKBlockedStart {
	332	+	close(o.startEntered)
	333	+	<-o.releaseStart
	334	+	}
	335	+	o.mu.Lock()
	336	+	o.marks = append(o.marks, mark)
	337	+	o.mu.Unlock()
	338	+	}
	339	+
	340	+	func (o *ackSpanInterleavingObserver) snapshot() []ports.RuntimeMark {
	341	+	o.mu.Lock()
	342	+	defer o.mu.Unlock()
	343	+	return append([]ports.RuntimeMark(nil), o.marks...)
	344	+	}
	345	+
	346	+	// An ACK can consume deferred work while the start mark is still waiting on
	347	+	// observer I/O. The End must wait for that Start publication, but the ACK
	348	+	// itself must not wait for either observer call.
	349	+	func TestTransportObservabilityACKBlockedStartPublishesBeforeEnd(t *testing.T) {
	350	+	observer := newACKSpanInterleavingObserver()
	351	+	ready := false
	352	+	rc := newRenderCoordinator(renderCoordinatorOptions{
	353	+	observer: observer,
	354	+	ackReady: func() bool { return ready },
	355	+	})
	356	+	rc.mu.Lock()
	357	+	rc.pending = true
	358	+	gen := rc.generation
	359	+	rc.mu.Unlock()
	360	+
	361	+	fireDone := make(chan struct{})
	362	+	go func() {
	363	+	rc.fire(gen, false, true)
	364	+	close(fireDone)
	365	+	}()
	366	+	awaitDaemonObserver(t, observer.startEntered, "ACK-blocked start observer")
	367	+
	368	+	ready = true
	369	+	ackDone := make(chan struct{})
	370	+	go func() {
	371	+	rc.notifyAck()
	372	+	close(ackDone)
	373	+	}()
	374	+	awaitDaemonObserver(t, ackDone, "nonblocking ACK progress")
	375	+	if marks := observer.snapshot(); len(marks) != 0 {
	376	+	t.Fatalf("ACK-blocked end published before its blocked start: %#v", marks)
	377	+	}
	378	+
	379	+	close(observer.releaseStart)
	380	+	awaitDaemonObserver(t, fireDone, "ACK-blocked start completion")
	381	+	marks := observer.snapshot()
	382	+	if len(marks) != 2 \|\| marks[0].Kind != ports.RuntimeACKBlockedStart \|\| marks[1].Kind != ports.RuntimeACKBlockedEnd {
	383	+	t.Fatalf("ACK-blocked marks = %#v, want one ordered start/end pair", marks)
	384	+	}
	385	+	if marks[0].Sequence != marks[1].Sequence \|\| marks[0].RequestID != marks[1].RequestID \|\| marks[0].Epoch != marks[1].Epoch {
	386	+	t.Fatalf("ACK-blocked span correlation changed: start=%+v end=%+v", marks[0], marks[1])
	387	+	}
	388	+	}

A testdata/perf/README.md +69 -0

@@ -0,0 +1,69 @@
	1	+	# S5a transport-observability evidence
	2	+
	3	+	`manifest.json` is the complete, measurement-only public-CLI matrix: four
	4	+	120x40 topologies (`1x4`, `4x1`, `4x4`, `8x1`), 10,000 history rows per pane,
	5	+	nine workloads, and seven transports (local, SSH stdio, UDP baseline, UDP 25 ms,
	6	+	UDP 100 ms, and UDP with 0% or 1% loss). It contains 252 scenario combinations;
	7	+	there are no omitted combinations. A genuinely unavailable combination must
	8	+	remain listed with `inapplicable_reason` and no launched roles.
	9	+
	10	+	Build and run the harness with at least a 10-second warmup, 30-second measured
	11	+	interval, and 10 independent repetitions:
	12	+
	13	+	```sh
	14	+	go build -o /tmp/vev-perf ./
	15	+	rm -rf testdata/perf/results
	16	+	go run ./cmd/vev-perf-harness --vev-bin /tmp/vev-perf \
	17	+	--manifest testdata/perf/manifest.json --out testdata/perf/results \
	18	+	--warmup 10s --duration 30s --repetitions 10
	19	+	```
	20	+
	21	+	Each run writes `manifest.json`, `raw-harness.jsonl`, one JSONL trace per
	22	+	launched process, `runs.json`, and `summary.json`. The process manifest records
	23	+	process ID, role, scenario/run, clock domain, exact exclusive trace path, and
	24	+	identity. Every launched role receives `VEV_PERF_TRACE`, `VEV_PERF_PROCESS_ID`,
	25	+	`VEV_PERF_SCENARIO`, and `VEV_PERF_RUN`; the first is exclusive to that process
	26	+	and the remaining values correlate it to the manifest. The harness owns only the end-to-end boundaries: input injection to
	27	+	terminal bytes successfully flushed. It never subtracts ticks from daemon,
	28	+	client, or adapter processes.
	29	+
	30	+	A process trace is JSONL with exactly these keys and types:
	31	+
	32	+	```json
	33	+	{"schema":1,"process_id":"p","component":"daemon","scenario":"s","run":1,"sequence":1,"request_id":1,"epoch":1,"kind":"diff_start","tick":0,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	34	+	```
	35	+
	36	+	`tick` is owned and stamped by that process's injected monotonic clock. Producers
	37	+	must leave it zero; one observer/clock is shared by components in a process.
	38	+	Ticks from different `process_id` or clock domains are never ordered or
	39	+	subtracted. Correlation uses scenario, run, sequence, request ID, and epoch.
	40	+
	41	+	Named process-local spans are: capture, compose, diff (`diff_start/end`), queue
	42	+	wait (`queue_enqueued/dequeued`), ACK-blocked interval, emit, adapter send,
	43	+	and adapter receive. Adapter spans are reported by component/transport; bytes,
	44	+	fragments, retransmits, pending, and ACK RTT remain diagnostics. Reports retain
	45	+	raw records and counts plus p50/p95/p99 (and max where applicable). End-to-end
	46	+	reports are harness-clock distributions only.
	47	+
	48	+	The harness injects at most one measured event per second and requires at least
	49	+	10 complete in-interval event pairs. Both the input-injection and successful
	50	+	terminal-flush boundaries must be within the measured interval; warmup and
	51	+	straddling pairs are excluded from percentile samples (but retained as raw
	52	+	evidence). Reject a result for missing or unmatched boundaries, missing flush,
	53	+	invalid correlation, nonmonotonic same-process sequence, negative same-domain
	54	+	duration, fewer than 10 complete in-interval pairs, invalid denominators,
	55	+	cross-process tick math, less than 30 seconds measured, or fewer than 10
	56	+	repetitions. Warmup is excluded from measured samples. Raw traces are evidence, not a transport-policy or
	57	+	optimization gate; wire bytes, protocol version, rendering policy, pacing,
	58	+	compression, and loss behavior are unchanged.
	59	+
	60	+	## Bounded manifest and harness checks
	61	+
	62	+	Before a measurement, validate the manifest shape and run the harness tests without launching the full matrix:
	63	+
	64	+	```sh
	65	+	jq -e '(.topologies\|length)==4 and (.workloads\|length)==9 and (.transports\|length)==7 and (.scenarios\|length)==252' testdata/perf/manifest.json
	66	+	go test ./cmd/vev-perf-harness
	67	+	```
	68	+
	69	+	The full command above is the measurement procedure; its 10-second warmup, 30-second measured interval, and 10 repetitions are the minimum accepted settings. For a bounded local evidence run, add `--scenario 1x4-idle-local`; the harness validates the complete canonical manifest before it applies that selection, so a single-scenario run cannot hide missing matrix coverage.

A testdata/perf/manifest.json +2808 -0 large diff (2808 lines changed): collapsed

load diff