Merge pull request #78 from bnema/perf/transport-observability

perf(obs): add transport runtime measurement harness

commit: f106d97fc96812b53243cdc4fc249d9e6bcdae85
author: bnema · 3mo ago · 2026-07-13 14:24 UTC
committer: GitHub · 3mo ago · 2026-07-13 14:24 UTC
merge of: 187a82b + 6b1947d · diff against the first parent
signature: GPG UnverifiedB5690EEEBB952194 The key is unknown

46 files · +9917 -201

A cmd/vev-perf-harness/evidence.go +324 -0

@@ -0,0 +1,324 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"os"
	9	+	"sort"
	10	+	"strings"
	11	+	)
	12	+
	13	+	func measuredEventSamples(marks []harnessMark, interval measuredInterval) ([]eventSample, error) {
	14	+	if interval.End < interval.Start {
	15	+	return nil, errors.New("negative measured interval")
	16	+	}
	17	+	starts := map[uint64]int64{}
	18	+	events := make([]eventSample, 0, len(marks)/2)
	19	+	for _, m := range marks {
	20	+	if !m.Valid {
	21	+	return nil, errors.New("invalid harness boundary")
	22	+	}
	23	+	switch m.Kind {
	24	+	case "input_injected":
	25	+	if _, exists := starts[m.Sequence]; exists {
	26	+	return nil, errors.New("duplicate input boundary")
	27	+	}
	28	+	starts[m.Sequence] = m.Tick
	29	+	case "terminal_flushed":
	30	+	start, ok := starts[m.Sequence]
	31	+	if !ok {
	32	+	return nil, errors.New("terminal flush without input pair")
	33	+	}
	34	+	if m.Tick < start {
	35	+	return nil, errors.New("negative harness duration")
	36	+	}
	37	+	delete(starts, m.Sequence)
	38	+	if start >= interval.Start && m.Tick < interval.End {
	39	+	events = append(events, eventSample{Sequence: m.Sequence, Injected: start, Flushed: m.Tick, Latency: m.Tick - start})
	40	+	}
	41	+	default:
	42	+	return nil, errors.New("unknown harness mark")
	43	+	}
	44	+	}
	45	+	if len(starts) != 0 {
	46	+	return nil, errors.New("missing terminal flush pair")
	47	+	}
	48	+	sort.Slice(events, func(i, j int) bool {
	49	+	if events[i].Injected == events[j].Injected {
	50	+	return events[i].Sequence < events[j].Sequence
	51	+	}
	52	+	return events[i].Injected < events[j].Injected
	53	+	})
	54	+	return events, nil
	55	+	}
	56	+
	57	+	func requireMinimumEventSamples(events []eventSample) error {
	58	+	if len(events) < minimumInIntervalEventSamples {
	59	+	return fmt.Errorf("insufficient in-interval event samples: got %d, need at least %d", len(events), minimumInIntervalEventSamples)
	60	+	}
	61	+	return nil
	62	+	}
	63	+
	64	+	func eventLatencies(events []eventSample) []int64 {
	65	+	out := make([]int64, len(events))
	66	+	for i, event := range events {
	67	+	out[i] = event.Latency
	68	+	}
	69	+	return out
	70	+	}
	71	+
	72	+	func eventCadenceSamples(events []eventSample) ([]int64, int64) {
	73	+	if len(events) < 2 {
	74	+	return nil, 0
	75	+	}
	76	+	gaps := make([]int64, 0, len(events)-1)
	77	+	var maxGap int64
	78	+	for i := 1; i < len(events); i++ {
	79	+	gap := events[i].Injected - events[i-1].Injected
	80	+	gaps = append(gaps, gap)
	81	+	if gap > maxGap {
	82	+	maxGap = gap
	83	+	}
	84	+	}
	85	+	return gaps, maxGap
	86	+	}
	87	+
	88	+	func eventCadence(events []eventSample) (distribution, int64) {
	89	+	gaps, maxGap := eventCadenceSamples(events)
	90	+	if len(gaps) == 0 {
	91	+	return distribution{}, maxGap
	92	+	}
	93	+	return percentiles(gaps), maxGap
	94	+	}
	95	+
	96	+	func pairedSamples(marks []harnessMark) ([]int64, error) {
	97	+	starts := map[uint64]int64{}
	98	+	var out []int64
	99	+	for _, m := range marks {
	100	+	if !m.Valid {
	101	+	return nil, errors.New("invalid harness boundary")
	102	+	}
	103	+	switch m.Kind {
	104	+	case "input_injected":
	105	+	if _, ok := starts[m.Sequence]; ok {
	106	+	return nil, errors.New("duplicate input boundary")
	107	+	}
	108	+	starts[m.Sequence] = m.Tick
	109	+	case "terminal_flushed":
	110	+	start, ok := starts[m.Sequence]
	111	+	if !ok {
	112	+	return nil, errors.New("terminal flush without input pair")
	113	+	}
	114	+	if m.Tick < start {
	115	+	return nil, errors.New("negative harness duration")
	116	+	}
	117	+	out = append(out, m.Tick-start)
	118	+	delete(starts, m.Sequence)
	119	+	default:
	120	+	return nil, errors.New("unknown harness mark")
	121	+	}
	122	+	}
	123	+	if len(starts) != 0 {
	124	+	return nil, errors.New("missing terminal flush pair")
	125	+	}
	126	+	if len(out) == 0 {
	127	+	return nil, errors.New("insufficient harness samples")
	128	+	}
	129	+	return out, nil
	130	+	}
	131	+
	132	+	// traceRecord is intentionally local to the command. It mirrors JSONL fields
	133	+	// needed for correlation but never exposes a clock to any production component.
	134	+	type traceRecord struct {
	135	+	ProcessID string `json:"process_id"`
	136	+	Component string `json:"component"`
	137	+	Scenario string `json:"scenario"`
	138	+	Run uint64 `json:"run"`
	139	+	Sequence uint64 `json:"sequence"`
	140	+	RequestID uint64 `json:"request_id"`
	141	+	Epoch uint64 `json:"epoch"`
	142	+	Kind string `json:"kind"`
	143	+	Tick int64 `json:"tick"`
	144	+	Valid bool `json:"valid"`
	145	+	}
	146	+
	147	+	type spanPair struct{ start, end, name string }
	148	+
	149	+	// mergeProcessTraces permits only records from one manifest process in a span.
	150	+	// In particular, the correlation key includes process_id before ticks are read.
	151	+	func mergeProcessTraces(mappings []processMapping) ([]span, error) {
	152	+	known := map[string]processMapping{}
	153	+	for _, m := range mappings {
	154	+	if m.ProcessID == "" \|\| m.ClockDomain == "" \|\| m.TracePath == "" \|\| known[m.ProcessID].ProcessID != "" {
	155	+	return nil, errors.New("duplicate or incomplete manifest process mapping")
	156	+	}
	157	+	known[m.ProcessID] = m
	158	+	}
	159	+	out := []span{}
	160	+	for _, m := range mappings {
	161	+	f, err := os.Open(m.TracePath)
	162	+	if err != nil {
	163	+	return nil, fmt.Errorf("open manifest trace for %s role: %w", m.Role, err)
	164	+	}
	165	+	var records []traceRecord
	166	+	scan := bufio.NewScanner(f)
	167	+	for scan.Scan() {
	168	+	var r traceRecord
	169	+	if err := json.Unmarshal(scan.Bytes(), &r); err != nil {
	170	+	return nil, closeFile(f, err)
	171	+	}
	172	+	if r.ProcessID != m.ProcessID \|\| r.Scenario != m.Scenario \|\| r.Run != uint64(m.Run) {
	173	+	return nil, closeFile(f, errors.New("trace record identity does not match manifest"))
	174	+	}
	175	+	if r.Scenario == "" \|\| r.Run == 0 \|\| r.Component == "" \|\| r.Sequence == 0 \|\| r.RequestID == 0 \|\| r.Epoch == 0 {
	176	+	return nil, closeFile(f, errors.New("trace record has invalid correlation fields"))
	177	+	}
	178	+	if _, ok := known[r.ProcessID]; !ok {
	179	+	return nil, closeFile(f, errors.New("unknown trace process_id"))
	180	+	}
	181	+	records = append(records, r)
	182	+	}
	183	+	if err := scan.Err(); err != nil {
	184	+	return nil, closeFile(f, err)
	185	+	}
	186	+	if err := f.Close(); err != nil {
	187	+	return nil, err
	188	+	}
	189	+	// A process shares one observer across concurrent components. Correlation
	190	+	// IDs are allocated before a goroutine reaches its first mark, so a later
	191	+	// ID can be serialized before an earlier one. Their first appearance is
	192	+	// consequently not an ordering contract. Only marks in the same component
	193	+	// and exact correlation domain may pair, and each pair's ticks must retain
	194	+	// its own start-before-end ordering.
	195	+	starts := map[string]traceRecord{}
	196	+	for _, r := range records {
	197	+	for _, pair := range spanPairs {
	198	+	key := fmt.Sprintf("%s/%s/%s/%d/%d/%d/%d", r.ProcessID, r.Component, m.Scenario, m.Run, r.Sequence, r.RequestID, r.Epoch)
	199	+	if r.Kind == pair.start {
	200	+	k := pair.name + "/" + key
	201	+	if _, exists := starts[k]; exists {
	202	+	return nil, errors.New("duplicate process-local span start")
	203	+	}
	204	+	starts[k] = r
	205	+	}
	206	+	if r.Kind == pair.end {
	207	+	k := pair.name + "/" + key
	208	+	start, ok := starts[k]
	209	+	if !ok {
	210	+	return nil, errors.New("span end without same-process start")
	211	+	}
	212	+	if r.Tick < start.Tick {
	213	+	return nil, errors.New("negative same-process span")
	214	+	}
	215	+	// Validity affects only measurement eligibility, never structural
	216	+	// pairing: every start/end must still match in its exact process,
	217	+	// component, and correlation domain. A failed start, failed end, or
	218	+	// both is a paired diagnostic fact but contributes no latency sample.
	219	+	if start.Valid && r.Valid {
	220	+	out = append(out, span{Component: r.Component, Name: pair.name, Samples: []int64{r.Tick - start.Tick}})
	221	+	}
	222	+	delete(starts, k)
	223	+	}
	224	+	}
	225	+	}
	226	+	if len(starts) != 0 {
	227	+	keys := make([]string, 0, len(starts))
	228	+	for key := range starts {
	229	+	keys = append(keys, key)
	230	+	}
	231	+	sort.Strings(keys)
	232	+	return nil, fmt.Errorf("missing process-local span pair for %s: %s", m.ProcessID, strings.Join(keys, ", "))
	233	+	}
	234	+	}
	235	+	return out, nil
	236	+	}
	237	+
	238	+	func summarizeSpans(all []span) []spanSummary {
	239	+	grouped := map[string]*span{}
	240	+	for _, s := range all {
	241	+	k := s.Component + "\x00" + s.Name
	242	+	if grouped[k] == nil {
	243	+	grouped[k] = &span{Component: s.Component, Name: s.Name}
	244	+	}
	245	+	grouped[k].Samples = append(grouped[k].Samples, s.Samples...)
	246	+	}
	247	+	keys := make([]string, 0, len(grouped))
	248	+	for k := range grouped {
	249	+	keys = append(keys, k)
	250	+	}
	251	+	sort.Strings(keys)
	252	+	out := make([]spanSummary, 0, len(keys))
	253	+	for _, k := range keys {
	254	+	s := grouped[k]
	255	+	if len(s.Samples) < minimumRepetitions {
	256	+	out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: distribution{Count: len(s.Samples)}})
	257	+	continue
	258	+	}
	259	+	out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: percentiles(s.Samples)})
	260	+	}
	261	+	return out
	262	+	}
	263	+
	264	+	func percentiles(samples []int64) distribution {
	265	+	v := append([]int64(nil), samples...)
	266	+	sort.Slice(v, func(i, j int) bool { return v[i] < v[j] })
	267	+	at := func(p int) int64 { return v[(len(v)-1)*p/100] }
	268	+	return distribution{len(v), at(50), at(95), at(99), v[len(v)-1]}
	269	+	}
	270	+
	271	+	func closeFile(f *os.File, err error) error {
	272	+	return errors.Join(err, f.Close())
	273	+	}
	274	+
	275	+	func writeJSON(path string, v any) error {
	276	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
	277	+	if err != nil {
	278	+	return err
	279	+	}
	280	+	if err := json.NewEncoder(f).Encode(v); err != nil {
	281	+	return closeFile(f, err)
	282	+	}
	283	+	return f.Close()
	284	+	}
	285	+
	286	+	func mustJSON(v any) string {
	287	+	b, err := json.Marshal(v)
	288	+	if err != nil {
	289	+	panic(fmt.Sprintf("marshal JSON: %v", err))
	290	+	}
	291	+	return string(b)
	292	+	}
	293	+
	294	+	func safeName(s string) string {
	295	+	return strings.Map(func(r rune) rune {
	296	+	if r >= 'a' && r <= 'z' \|\| r >= 'A' && r <= 'Z' \|\| r >= '0' && r <= '9' \|\| r == '-' \|\| r == '_' {
	297	+	return r
	298	+	}
	299	+	return '-'
	300	+	}, s)
	301	+	}
	302	+
	303	+	func readJSONL(path string) ([]map[string]json.RawMessage, error) {
	304	+	f, err := os.Open(path)
	305	+	if err != nil {
	306	+	return nil, err
	307	+	}
	308	+	var r []map[string]json.RawMessage
	309	+	scan := bufio.NewScanner(f)
	310	+	for scan.Scan() {
	311	+	var v map[string]json.RawMessage
	312	+	if err := json.Unmarshal(scan.Bytes(), &v); err != nil {
	313	+	return nil, closeFile(f, err)
	314	+	}
	315	+	r = append(r, v)
	316	+	}
	317	+	if err := scan.Err(); err != nil {
	318	+	return nil, closeFile(f, err)
	319	+	}
	320	+	if err := f.Close(); err != nil {
	321	+	return nil, err
	322	+	}
	323	+	return r, nil
	324	+	}

A cmd/vev-perf-harness/evidence_test.go +387 -0

@@ -0,0 +1,387 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"os"
	8	+	"path/filepath"
	9	+	"strings"
	10	+	"testing"
	11	+	"time"
	12	+	)
	13	+
	14	+	func TestCLITransportSeamOwnsExclusivePeerTraceAndCleanup(t *testing.T) {
	15	+	fixtures := []struct {
	16	+	name string
	17	+	tr transport
	18	+	rtt time.Duration
	19	+	loss int
	20	+	}{
	21	+	{"baseline", transport{ID: "udp_baseline", Kind: "udp"}, 0, 0},
	22	+	{"25ms", transport{ID: "udp_25ms", Kind: "udp", RTTMS: 25}, 25 * time.Millisecond, 0},
	23	+	{"100ms", transport{ID: "udp_100ms", Kind: "udp", RTTMS: 100}, 100 * time.Millisecond, 0},
	24	+	{"loss0", transport{ID: "udp_loss_0pct", Kind: "udp"}, 0, 0},
	25	+	{"loss1", transport{ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1}, 0, 1},
	26	+	}
	27	+	for _, tc := range fixtures {
	28	+	t.Run(tc.name, func(t *testing.T) {
	29	+	dir := t.TempDir()
	30	+	if err := os.Chmod(dir, 0o700); err != nil {
	31	+	t.Fatal(err)
	32	+	}
	33	+	var configs []udpNetemConfig
	34	+	netem := &fakeUDPNetem{port: 45678}
	35	+	l := &cliLauncher{bin: "/bin/true", netemFactory: func(c udpNetemConfig) (udpNetem, error) {
	36	+	configs = append(configs, c)
	37	+	return netem, nil
	38	+	}}
	39	+	m := processMapping{ProcessID: "udp-peer", TracePath: filepath.Join(dir, "udp.jsonl"), Role: "udp_peer"}
	40	+	if err := os.WriteFile(m.TracePath, nil, 0o600); err != nil {
	41	+	t.Fatal(err)
	42	+	}
	43	+	p, err := l.preparePeer(m, roleCommand{Args: []string{"_udp-proxy", "work"}, Transport: tc.tr})
	44	+	if err != nil {
	45	+	t.Fatal(err)
	46	+	}
	47	+	if len(configs) != 1 \|\| configs[0].RTT != tc.rtt \|\| configs[0].LossPercent != tc.loss \|\| configs[0].TargetPath != filepath.Join(dir, "udp-peer.target") {
	48	+	t.Fatalf("fixture did not reach netem seam: %+v", configs)
	49	+	}
	50	+	shim, err := os.ReadFile(filepath.Join(dir, "ssh"))
	51	+	if err != nil {
	52	+	t.Fatal(err)
	53	+	}
	54	+	text := string(shim)
	55	+	for _, want := range []string{"_udp-proxy", m.TracePath, m.ProcessID, "udp-peer.target", "VEV-UDP %s %s\\n' 45678", "udp-peer.pid"} {
	56	+	if !strings.Contains(text, want) {
	57	+	t.Errorf("seam does not retain %q:\n%s", want, text)
	58	+	}
	59	+	}
	60	+	if strings.Contains(text, "VEV_PERF_UDP_") {
	61	+	t.Fatalf("ignored vev UDP environment was used instead of netem: %s", text)
	62	+	}
	63	+	// A nonexistent pid is already-cleaned-up; Close must still close the
	64	+	// harness-owned emulator.
	65	+	if err := p.Close(); err != nil {
	66	+	t.Fatal(err)
	67	+	}
	68	+	if !netem.closed {
	69	+	t.Fatal("harness netem was not cleaned up")
	70	+	}
	71	+	})
	72	+	}
	73	+	}
	74	+
	75	+	func TestHarnessRejectsInvalidBoundaryPairs(t *testing.T) {
	76	+	for _, marks := range [][]harnessMark{
	77	+	{{Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
	78	+	{{Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}, {Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
	79	+	{{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}},
	80	+	{{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}},
	81	+	} {
	82	+	if _, err := pairedSamples(marks); err == nil {
	83	+	t.Fatalf("invalid marks accepted: %+v", marks)
	84	+	}
	85	+	}
	86	+	got, err := pairedSamples([]harnessMark{{Sequence: 9, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 9, Kind: "terminal_flushed", Tick: 5, Valid: true}})
	87	+	if err != nil \|\| len(got) != 1 \|\| got[0] != 4 {
	88	+	t.Fatalf("got %v %v", got, err)
	89	+	}
	90	+	}
	91	+
	92	+	func TestHarnessCollectsOnlyCompleteMeasuredIntervalEvents(t *testing.T) {
	93	+	interval := measuredInterval{Start: 100, End: 200}
	94	+	marks := []harnessMark{
	95	+	{Sequence: 1, Kind: "input_injected", Tick: 10, Valid: true},
	96	+	{Sequence: 1, Kind: "terminal_flushed", Tick: 20, Valid: true}, // warmup
	97	+	{Sequence: 2, Kind: "input_injected", Tick: 100, Valid: true},
	98	+	{Sequence: 2, Kind: "terminal_flushed", Tick: 120, Valid: true},
	99	+	{Sequence: 3, Kind: "input_injected", Tick: 150, Valid: true},
	100	+	{Sequence: 3, Kind: "terminal_flushed", Tick: 201, Valid: true}, // exits interval
	101	+	{Sequence: 4, Kind: "input_injected", Tick: 99, Valid: true},
	102	+	{Sequence: 4, Kind: "terminal_flushed", Tick: 110, Valid: true}, // enters interval
	103	+	{Sequence: 5, Kind: "input_injected", Tick: 180, Valid: true},
	104	+	{Sequence: 5, Kind: "terminal_flushed", Tick: 200, Valid: true}, // end is excluded
	105	+	}
	106	+	events, err := measuredEventSamples(marks, interval)
	107	+	if err != nil {
	108	+	t.Fatal(err)
	109	+	}
	110	+	if len(events) != 1 \|\| events[0].Sequence != 2 \|\| events[0].Latency != 20 {
	111	+	t.Fatalf("measured events=%+v, want only complete sequence 2", events)
	112	+	}
	113	+	}
	114	+
	115	+	func TestHarnessRejectsInsufficientMeasuredEventSamples(t *testing.T) {
	116	+	marks := make([]harnessMark, 0, 2*(minimumInIntervalEventSamples-1))
	117	+	for i := 0; i < minimumInIntervalEventSamples-1; i++ {
	118	+	marks = append(marks,
	119	+	harnessMark{Sequence: uint64(i + 1), Kind: "input_injected", Tick: int64(i * 100), Valid: true},
	120	+	harnessMark{Sequence: uint64(i + 1), Kind: "terminal_flushed", Tick: int64(i*100 + 10), Valid: true},
	121	+	)
	122	+	}
	123	+	events, err := measuredEventSamples(marks, measuredInterval{Start: 0, End: int64(minimumInIntervalEventSamples * 100)})
	124	+	if err != nil {
	125	+	t.Fatal(err)
	126	+	}
	127	+	if err := requireMinimumEventSamples(events); err == nil {
	128	+	t.Fatal("insufficient measured event samples accepted")
	129	+	}
	130	+	}
	131	+
	132	+	func TestHarnessRunRejectsStraddlingEventsAsInsufficient(t *testing.T) {
	133	+	dir := t.TempDir()
	134	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	135	+	if err != nil {
	136	+	t.Fatal(err)
	137	+	}
	138	+	t.Cleanup(func() {
	139	+	if err := raw.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
	140	+	t.Error(err)
	141	+	}
	142	+	})
	143	+	clock := &fakeClock{}
	144	+	h := defaultHarness()
	145	+	h.clock, h.launcher = clock, insufficientEventLauncher{clock: clock}
	146	+	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "late", Roles: []string{"daemon", "client"}}, 1, raw)
	147	+	if err == nil \|\| !strings.Contains(err.Error(), "insufficient in-interval event samples") {
	148	+	t.Fatalf("straddling run error=%v, want insufficient in-interval sample rejection", err)
	149	+	}
	150	+	if err := raw.Close(); err != nil {
	151	+	t.Fatal(err)
	152	+	}
	153	+	marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
	154	+	if got, want := len(marks), 4; got != want {
	155	+	t.Fatalf("raw marks=%d, want warmup and straddling pair=%d: %+v", got, want, marks)
	156	+	}
	157	+	if straddling := marks[len(marks)-1]; straddling.Sequence != 2 \|\| straddling.Kind != "terminal_flushed" \|\| !straddling.Valid {
	158	+	t.Fatalf("straddling boundary=%+v", straddling)
	159	+	}
	160	+	}
	161	+
	162	+	func TestHarnessCalculatesEventCadenceAndMaxGapFromRawPairs(t *testing.T) {
	163	+	events := []eventSample{
	164	+	{Sequence: 1, Injected: 100, Flushed: 110, Latency: 10},
	165	+	{Sequence: 2, Injected: 130, Flushed: 140, Latency: 10},
	166	+	{Sequence: 3, Injected: 180, Flushed: 190, Latency: 10},
	167	+	}
	168	+	cadence, maxGap := eventCadence(events)
	169	+	if cadence.Count != 2 \|\| cadence.P50 != 30 \|\| cadence.P95 != 30 \|\| cadence.P99 != 30 \|\| maxGap != 50 {
	170	+	t.Fatalf("cadence=%+v maxGap=%d", cadence, maxGap)
	171	+	}
	172	+	}
	173	+
	174	+	func TestHarnessIncludesAllRequiredProcessLocalSpans(t *testing.T) {
	175	+	got := make([]string, 0, len(spanPairs))
	176	+	for _, pair := range spanPairs {
	177	+	got = append(got, pair.name)
	178	+	}
	179	+	want := []string{"capture_duration", "compose_duration", "diff_duration", "queue_wait", "ack_blocked_interval", "emit_duration", "adapter_send_duration", "adapter_receive_duration"}
	180	+	if !equalStrings(got, want) {
	181	+	t.Fatalf("span pairs = %v, want %v", got, want)
	182	+	}
	183	+	}
	184	+
	185	+	func TestHarnessRawRecordsAreDeterministicJSONL(t *testing.T) {
	186	+	var b bytes.Buffer
	187	+	for _, m := range []harnessMark{{Scenario: "s", Run: 1, Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Scenario: "s", Run: 1, Sequence: 1, Kind: "terminal_flushed", Tick: 2, Valid: true}} {
	188	+	b.WriteString(mustJSON(m))
	189	+	b.WriteByte('\n')
	190	+	}
	191	+	path := filepath.Join(t.TempDir(), "raw.jsonl")
	192	+	if err := os.WriteFile(path, b.Bytes(), 0o600); err != nil {
	193	+	t.Fatal(err)
	194	+	}
	195	+	if got, err := readJSONL(path); err != nil \|\| len(got) != 2 {
	196	+	t.Fatalf("records=%d err=%v", len(got), err)
	197	+	}
	198	+	}
	199	+
	200	+	func TestHarnessClosesRolesBeforeMergingReceiveSpans(t *testing.T) {
	201	+	dir := t.TempDir()
	202	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	203	+	if err != nil {
	204	+	t.Fatal(err)
	205	+	}
	206	+	t.Cleanup(func() {
	207	+	if err := raw.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
	208	+	t.Error(err)
	209	+	}
	210	+	})
	211	+	launcher := &closeTraceLauncher{}
	212	+	h := defaultHarness()
	213	+	h.clock, h.launcher = &fakeClock{}, launcher
	214	+	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "receive-cleanup", Roles: []string{"daemon", "client"}}, 1, raw)
	215	+	if err != nil {
	216	+	t.Fatal(err)
	217	+	}
	218	+	if len(result.Spans) != 2 {
	219	+	t.Fatalf("post-cleanup receive spans=%+v", result.Spans)
	220	+	}
	221	+	for _, s := range result.Spans {
	222	+	if s.Name != "adapter_receive_duration" \|\| len(s.Samples) != 1 \|\| s.Samples[0] != 10 {
	223	+	t.Fatalf("unmatched or wrong receive span: %+v", s)
	224	+	}
	225	+	}
	226	+	for i, p := range launcher.process {
	227	+	if !p.closed {
	228	+	t.Fatalf("role %s was not closed before trace merge", launcher.mappings[i].Role)
	229	+	}
	230	+	}
	231	+	}
	232	+
	233	+	func TestHarnessWritesRequiredEvidenceWithSufficientSpans(t *testing.T) {
	234	+	base, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
	235	+	if err != nil {
	236	+	t.Fatal(err)
	237	+	}
	238	+	for i := range base.Scenarios {
	239	+	if i > 0 {
	240	+	base.Scenarios[i].InapplicableReason = "fixture limits this test to one public topology"
	241	+	base.Scenarios[i].Roles = nil
	242	+	}
	243	+	}
	244	+	dir := t.TempDir()
	245	+	manifestPath := filepath.Join(dir, "manifest.json")
	246	+	if err := os.WriteFile(manifestPath, []byte(mustJSON(base)), 0o600); err != nil {
	247	+	t.Fatal(err)
	248	+	}
	249	+	l := &traceLauncher{}
	250	+	h := defaultHarness()
	251	+	h.clock = &fakeClock{}
	252	+	h.launcher = l
	253	+	h.gitSHA = func() string { return "test-sha" }
	254	+	if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", filepath.Join(dir, "out"), "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
	255	+	t.Fatal(err)
	256	+	}
	257	+	for _, name := range []string{"raw-harness.jsonl", "runs.json", "summary.json"} {
	258	+	if _, err := os.Stat(filepath.Join(dir, "out", name)); err != nil {
	259	+	t.Errorf("%s: %v", name, err)
	260	+	}
	261	+	}
	262	+	var got summary
	263	+	b, err := os.ReadFile(filepath.Join(dir, "out", "summary.json"))
	264	+	if err != nil {
	265	+	t.Fatal(err)
	266	+	}
	267	+	if err := json.Unmarshal(b, &got); err != nil {
	268	+	t.Fatal(err)
	269	+	}
	270	+	if got.EndToEnd.Count < 10*minimumInIntervalEventSamples \|\| got.Repetitions != 10 \|\| got.RunP50Dispersion.Count != 10 \|\| got.Cadence.Count == 0 \|\| got.MaxGap == 0 \|\| len(got.Spans) == 0 {
	271	+	t.Fatalf("summary=%+v", got)
	272	+	}
	273	+	for _, s := range got.Spans {
	274	+	if s.Distribution.Count < 10 {
	275	+	t.Fatalf("insufficient span summary: %+v", s)
	276	+	}
	277	+	}
	278	+	}
	279	+
	280	+	func TestHarnessAcceptsConcurrentProductionTraceInterleaving(t *testing.T) {
	281	+	path := filepath.Join(t.TempDir(), "local-client.jsonl")
	282	+	if err := os.WriteFile(path, []byte(boundedLocalConcurrentTrace), 0o600); err != nil {
	283	+	t.Fatal(err)
	284	+	}
	285	+	spans, err := mergeProcessTraces([]processMapping{{
	286	+	ProcessID: "1x4-idle-local-r002-client-02", ClockDomain: "1x4-idle-local-r002-client-02", TracePath: path,
	287	+	Scenario: "1x4-idle-local", Run: 2,
	288	+	}})
	289	+	if err != nil {
	290	+	t.Fatalf("valid concurrent production trace rejected: %v", err)
	291	+	}
	292	+	if len(spans) != 2 \|\| spans[0].Component != "ipc" \|\| spans[0].Name != "adapter_send_duration" \|\| spans[0].Samples[0] != 21900 \|\| spans[1].Component != "ipc" \|\| spans[1].Name != "adapter_receive_duration" \|\| spans[1].Samples[0] != 178550 {
	293	+	t.Fatalf("spans=%+v", spans)
	294	+	}
	295	+	}
	296	+
	297	+	func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {
	298	+	write := func(t *testing.T, records ...traceRecord) processMapping {
	299	+	t.Helper()
	300	+	path := filepath.Join(t.TempDir(), "trace.jsonl")
	301	+	f, err := os.Create(path)
	302	+	if err != nil {
	303	+	t.Fatal(err)
	304	+	}
	305	+	for _, r := range records {
	306	+	b, err := json.Marshal(r)
	307	+	if err != nil {
	308	+	t.Fatal(err)
	309	+	}
	310	+	if _, err := f.Write(append(b, '\n')); err != nil {
	311	+	t.Fatal(err)
	312	+	}
	313	+	}
	314	+	if err := f.Close(); err != nil {
	315	+	t.Fatal(err)
	316	+	}
	317	+	return processMapping{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}
	318	+	}
	319	+	base := func(kind string, tick int64) traceRecord {
	320	+	return traceRecord{ProcessID: "one", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: true}
	321	+	}
	322	+	for _, records := range [][]traceRecord{
	323	+	{base("diff_end", 1)},
	324	+	{base("diff_start", 2), base("diff_end", 1)},
	325	+	{base("diff_start", 1)},
	326	+	{base("diff_start", 1), base("diff_start", 2)},
	327	+	{base("diff_start", 1), func() traceRecord { r := base("diff_end", 2); r.Component = "ipc"; return r }()},
	328	+	{func() traceRecord { r := base("diff_start", 1); r.ProcessID = "other"; return r }()},
	329	+	{func() traceRecord { r := base("diff_start", 1); r.Scenario = "other-scenario"; return r }()},
	330	+	{func() traceRecord { r := base("diff_start", 1); r.Run = 2; return r }()},
	331	+	} {
	332	+	if _, err := mergeProcessTraces([]processMapping{write(t, records...)}); err == nil {
	333	+	t.Fatalf("invalid trace accepted: %+v", records)
	334	+	}
	335	+	}
	336	+	m := write(t, base("diff_start", 1), base("diff_end", 3))
	337	+	spans, err := mergeProcessTraces([]processMapping{m})
	338	+	if err != nil \|\| len(spans) != 1 \|\| spans[0].Samples[0] != 2 {
	339	+	t.Fatalf("spans=%+v err=%v", spans, err)
	340	+	}
	341	+	}
	342	+
	343	+	func TestHarnessExcludesFailedSpanDurationsWhileValidatingPairing(t *testing.T) {
	344	+	base := func(kind string, tick int64, valid bool) traceRecord {
	345	+	return traceRecord{ProcessID: "one", Component: "adapter", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: valid}
	346	+	}
	347	+	for _, tc := range []struct {
	348	+	name string
	349	+	records []traceRecord
	350	+	wantSpans int
	351	+	wantErr bool
	352	+	}{
	353	+	{"successful adapter send is sampled", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, true)}, 1, false},
	354	+	{"failed adapter send end is excluded", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, false)}, 0, false},
	355	+	{"failed adapter receive end is excluded", []traceRecord{base("adapter_receive_start", 10, true), base("adapter_receive_end", 20, false)}, 0, false},
	356	+	{"failed adapter send start is excluded", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_end", 20, true)}, 0, false},
	357	+	{"failed adapter receive boundaries are excluded", []traceRecord{base("adapter_receive_start", 10, false), base("adapter_receive_end", 20, false)}, 0, false},
	358	+	{"failed end without start is rejected", []traceRecord{base("adapter_send_end", 20, false)}, 0, true},
	359	+	{"duplicate failed start is rejected", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_start", 15, false)}, 0, true},
	360	+	{"negative failed span is rejected", []traceRecord{base("adapter_receive_start", 20, false), base("adapter_receive_end", 10, false)}, 0, true},
	361	+	{"cross-component failed end is rejected", []traceRecord{base("adapter_send_start", 10, true), func() traceRecord { r := base("adapter_send_end", 20, false); r.Component = "other"; return r }()}, 0, true},
	362	+	} {
	363	+	t.Run(tc.name, func(t *testing.T) {
	364	+	path := filepath.Join(t.TempDir(), "trace.jsonl")
	365	+	var lines []string
	366	+	for _, record := range tc.records {
	367	+	lines = append(lines, mustJSON(record))
	368	+	}
	369	+	if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
	370	+	t.Fatal(err)
	371	+	}
	372	+	spans, err := mergeProcessTraces([]processMapping{{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}})
	373	+	if tc.wantErr {
	374	+	if err == nil {
	375	+	t.Fatalf("invalid trace accepted: %+v", tc.records)
	376	+	}
	377	+	return
	378	+	}
	379	+	if err != nil \|\| len(spans) != tc.wantSpans {
	380	+	t.Fatalf("spans=%+v err=%v, want %d spans", spans, err, tc.wantSpans)
	381	+	}
	382	+	if tc.wantSpans == 1 && spans[0].Samples[0] != 10 {
	383	+	t.Fatalf("successful span=%+v, want 10ns", spans[0])
	384	+	}
	385	+	})
	386	+	}
	387	+	}

A cmd/vev-perf-harness/launcher.go +752 -0

@@ -0,0 +1,752 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"context"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"os"
	10	+	"os/exec"
	11	+	"path/filepath"
	12	+	"strconv"
	13	+	"strings"
	14	+	"sync"
	15	+	"syscall"
	16	+	"time"
	17	+
	18	+	"github.com/bnema/vev/pkg/safedir"
	19	+	"golang.org/x/sys/unix"
	20	+	)
	21	+
	22	+	type launchedProcess interface {
	23	+	Warmup([]byte) error
	24	+	Measure([]byte, func() error, func() error) error
	25	+	Close() error
	26	+	}
	27	+
	28	+	// roleCommand contains only argv accepted by app.parseArgs. Transport routing
	29	+	// metadata configures the harness's public-command seam; it is never passed as
	30	+	// an unrecognised vev flag.
	31	+	type roleCommand struct {
	32	+	Args []string
	33	+	Transport transport
	34	+	}
	35	+
	36	+	type launcher interface {
	37	+	Launch(processMapping, roleCommand) (launchedProcess, error)
	38	+	}
	39	+
	40	+	type launchedRole struct {
	41	+	mapping processMapping
	42	+	process launchedProcess
	43	+	}
	44	+
	45	+	// closeLaunchedRoles closes every role in reverse launch order. errors.Join
	46	+	// retains every cleanup failure in that stable order.
	47	+	func closeLaunchedRoles(processes []launchedRole) error {
	48	+	var failures []error
	49	+	for i := len(processes) - 1; i >= 0; i-- {
	50	+	if err := processes[i].process.Close(); err != nil {
	51	+	failures = append(failures, fmt.Errorf("close %s role: %w", processes[i].mapping.Role, err))
	52	+	}
	53	+	}
	54	+	return errors.Join(failures...)
	55	+	}
	56	+
	57	+	// cliLauncher is deliberately limited to executing the supplied public vev
	58	+	// binary. Scenario orchestration never reaches into an in-process daemon.
	59	+	type cliLauncher struct {
	60	+	bin string
	61	+	netemFactory udpNetemFactory
	62	+	mu sync.Mutex
	63	+	peers map[string]peerRoute
	64	+	runtimes map[string]string
	65	+	}
	66	+
	67	+	type peerRoute struct {
	68	+	mapping processMapping
	69	+	command roleCommand
	70	+	pidPath string
	71	+	netem udpNetem
	72	+	}
	73	+
	74	+	type preparedPeer struct {
	75	+	route peerRoute
	76	+	cleanupRuntime func() error
	77	+	}
	78	+
	79	+	func (p preparedPeer) Warmup([]byte) error { return nil }
	80	+
	81	+	func (p preparedPeer) Measure([]byte, func() error, func() error) error { return nil }
	82	+
	83	+	func (p preparedPeer) Close() error {
	84	+	var errs []error
	85	+	if p.route.pidPath != "" {
	86	+	b, err := os.ReadFile(p.route.pidPath)
	87	+	if err == nil {
	88	+	pid, parseErr := strconv.Atoi(strings.TrimSpace(string(b)))
	89	+	if parseErr != nil \|\| pid <= 0 {
	90	+	errs = append(errs, fmt.Errorf("invalid owned peer pid: %q", b))
	91	+	} else if killErr := syscall.Kill(pid, syscall.SIGTERM); killErr != nil && !errors.Is(killErr, syscall.ESRCH) {
	92	+	errs = append(errs, killErr)
	93	+	}
	94	+	} else if !errors.Is(err, os.ErrNotExist) {
	95	+	errs = append(errs, err)
	96	+	}
	97	+	}
	98	+	if p.route.netem != nil {
	99	+	if err := p.route.netem.Close(); err != nil {
	100	+	errs = append(errs, err)
	101	+	}
	102	+	}
	103	+	if p.cleanupRuntime != nil {
	104	+	if err := p.cleanupRuntime(); err != nil {
	105	+	errs = append(errs, err)
	106	+	}
	107	+	}
	108	+	return errors.Join(errs...)
	109	+	}
	110	+
	111	+	type terminalOutput interface {
	112	+	io.Writer
	113	+	Sync() error
	114	+	Close() error
	115	+	}
	116	+
	117	+	type cliProcess struct {
	118	+	cmd *exec.Cmd
	119	+	pty io.ReadWriteCloser
	120	+	output terminalOutput
	121	+	chunks chan []byte
	122	+	done chan struct{}
	123	+	closed sync.Once
	124	+	waitErr chan error
	125	+	waitTimeout func() <-chan time.Time
	126	+	forceCleanup func()
	127	+	forceKill func()
	128	+	shutdown func() error
	129	+	cleanupRuntime func() error
	130	+	terminalReady bool
	131	+	readyPath string
	132	+	}
	133	+
	134	+	func (l *cliLauncher) Launch(m processMapping, role roleCommand) (launchedProcess, error) {
	135	+	if len(role.Args) == 0 {
	136	+	return nil, errors.New("public CLI role has no arguments")
	137	+	}
	138	+	if info, err := os.Stat(l.bin); err != nil {
	139	+	return nil, err
	140	+	} else if info.IsDir() {
	141	+	return nil, fmt.Errorf("public CLI binary %q is a directory", l.bin)
	142	+	}
	143	+	if m.Role == "ssh_stdio_peer" \|\| m.Role == "udp_peer" {
	144	+	return l.preparePeer(m, role)
	145	+	}
	146	+	cmd := exec.Command(l.bin, role.Args...)
	147	+	runDir := filepath.Dir(m.TracePath)
	148	+	runtimeDir, err := l.runtimeDir(runDir)
	149	+	if err != nil {
	150	+	return nil, err
	151	+	}
	152	+	// All role inputs are absolute paths resolved at harness startup. Do not set
	153	+	// a role working directory: daemon-owned subprocess cleanup may otherwise
	154	+	// treat the evidence directory as its working tree and remove a preallocated
	155	+	// trace while a later repetition is being merged.
	156	+	cmd.Env = append(withoutEnv(os.Environ(), "VEV", "VEV_PERF_TRACE", "VEV_PERF_PROCESS_ID", "VEV_PERF_SCENARIO", "VEV_PERF_RUN", "VEV_REMOTE_TRANSPORT", "XDG_RUNTIME_DIR", "XDG_STATE_HOME"), traceEnvironment(m)...)
	157	+	cmd.Env = append(cmd.Env, "XDG_RUNTIME_DIR="+runtimeDir,
	158	+	"XDG_STATE_HOME="+filepath.Join(runDir, "state"), "TERM=xterm-256color")
	159	+	if m.Role == "client" {
	160	+	l.mu.Lock()
	161	+	peer, routed := l.peers[runDir]
	162	+	l.mu.Unlock()
	163	+	if routed {
	164	+	mode, err := remoteMode(peer.command.Transport)
	165	+	if err != nil {
	166	+	return nil, err
	167	+	}
	168	+	cmd.Env = append(cmd.Env, "PATH="+runDir+":"+os.Getenv("PATH"), "VEV_REMOTE_TRANSPORT="+mode)
	169	+	}
	170	+	}
	171	+	if err := safedir.EnsurePrivate(filepath.Join(runDir, "state")); err != nil {
	172	+	return nil, err
	173	+	}
	174	+	p := &cliProcess{cmd: cmd, waitErr: make(chan error, 1), readyPath: filepath.Join(runtimeDir, "vev", "daemon.sock")}
	175	+	if m.Role == "daemon" {
	176	+	p.cleanupRuntime = func() error { return l.releaseRuntime(runDir) }
	177	+	}
	178	+	if m.Role == "client" {
	179	+	master, slave, err := openPTY()
	180	+	if err != nil {
	181	+	return nil, err
	182	+	}
	183	+	output, err := os.OpenFile(filepath.Join(filepath.Dir(m.TracePath), "terminal-output"), os.O_WRONLY\|os.O_CREATE\|os.O_APPEND, 0o600)
	184	+	if err != nil {
	185	+	_ = master.Close()
	186	+	_ = slave.Close()
	187	+	return nil, err
	188	+	}
	189	+	cmd.Stdin, cmd.Stdout, cmd.Stderr = slave, slave, slave
	190	+	// Ctty is the child descriptor number after Start remaps slave to stdin.
	191	+	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true, Setctty: true, Ctty: 0}
	192	+	if err := cmd.Start(); err != nil {
	193	+	_ = output.Close()
	194	+	_ = master.Close()
	195	+	_ = slave.Close()
	196	+	if p.cleanupRuntime != nil {
	197	+	_ = p.cleanupRuntime()
	198	+	}
	199	+	return nil, err
	200	+	}
	201	+	_ = slave.Close()
	202	+	p.pty, p.output, p.chunks, p.done = master, output, make(chan []byte, 32), make(chan struct{})
	203	+	go p.copyTerminal()
	204	+	} else {
	205	+	// Every launched role owns a dedicated process group, so the bounded
	206	+	// cleanup path never signals the harness/test runner's group.
	207	+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	208	+	devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0)
	209	+	if err != nil {
	210	+	return nil, err
	211	+	}
	212	+	cmd.Stdin, cmd.Stdout, cmd.Stderr = devNull, devNull, devNull
	213	+	if err := cmd.Start(); err != nil {
	214	+	_ = devNull.Close()
	215	+	if p.cleanupRuntime != nil {
	216	+	_ = p.cleanupRuntime()
	217	+	}
	218	+	return nil, err
	219	+	}
	220	+	_ = devNull.Close()
	221	+	}
	222	+	if m.Role == "daemon" {
	223	+	// Stop the foreground daemon through its public CLI before its bounded
	224	+	// process-group fallback. This lets blocked carriage operations close and
	225	+	// serialize their failed end marks instead of being cut off by SIGKILL.
	226	+	p.shutdown = func() error {
	227	+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	228	+	defer cancel()
	229	+	shutdown := exec.CommandContext(ctx, l.bin, "kill", "--daemon")
	230	+	shutdown.Env = withoutEnv(cmd.Env, "VEV_PERF_TRACE", "VEV_PERF_PROCESS_ID", "VEV_PERF_SCENARIO", "VEV_PERF_RUN")
	231	+	return shutdown.Run()
	232	+	}
	233	+	}
	234	+	go func() { p.waitErr <- cmd.Wait() }()
	235	+	return p, nil
	236	+	}
	237	+
	238	+	// runtimeDir returns one short, private runtime directory shared by the roles
	239	+	// of one run. Keeping the Unix socket outside deeply nested evidence output
	240	+	// avoids the platform socket-path limit without weakening run isolation.
	241	+	func (l *cliLauncher) runtimeDir(runDir string) (string, error) {
	242	+	l.mu.Lock()
	243	+	defer l.mu.Unlock()
	244	+	if path := l.runtimes[runDir]; path != "" {
	245	+	return path, nil
	246	+	}
	247	+	path, err := os.MkdirTemp("", "vev-harness-runtime-")
	248	+	if err != nil {
	249	+	return "", err
	250	+	}
	251	+	if err := safedir.EnsurePrivate(path); err != nil {
	252	+	_ = os.RemoveAll(path)
	253	+	return "", err
	254	+	}
	255	+	if l.runtimes == nil {
	256	+	l.runtimes = make(map[string]string)
	257	+	}
	258	+	l.runtimes[runDir] = path
	259	+	return path, nil
	260	+	}
	261	+
	262	+	func (l *cliLauncher) releaseRuntime(runDir string) error {
	263	+	l.mu.Lock()
	264	+	path := l.runtimes[runDir]
	265	+	delete(l.runtimes, runDir)
	266	+	l.mu.Unlock()
	267	+	if path == "" {
	268	+	return nil
	269	+	}
	270	+	return os.RemoveAll(path)
	271	+	}
	272	+
	273	+	// preparePeer installs a per-run ssh command seam. The ordinary public client
	274	+	// still executes its documented remote attach command; its ssh child invokes
	275	+	// only the parsed public _stdio or _udp-proxy command. This makes the peer
	276	+	// carrying client traffic the declared, exclusively traced role rather than a
	277	+	// separately launched but unused process.
	278	+	func (l *cliLauncher) preparePeer(m processMapping, role roleCommand) (launchedProcess, error) {
	279	+	runDir := filepath.Dir(m.TracePath)
	280	+	l.mu.Lock()
	281	+	runtimeDir := l.runtimes[runDir]
	282	+	l.mu.Unlock()
	283	+	peerOwnsRuntime := runtimeDir == ""
	284	+	if peerOwnsRuntime {
	285	+	var err error
	286	+	runtimeDir, err = l.runtimeDir(runDir)
	287	+	if err != nil {
	288	+	return nil, err
	289	+	}
	290	+	}
	291	+	if err := safedir.EnsurePrivate(runDir); err != nil {
	292	+	return nil, err
	293	+	}
	294	+	if err := safedir.EnsurePrivate(runtimeDir); err != nil {
	295	+	return nil, err
	296	+	}
	297	+	if err := safedir.EnsurePrivate(filepath.Join(runDir, "state")); err != nil {
	298	+	return nil, err
	299	+	}
	300	+	if len(role.Args) != 2 \|\| (role.Args[0] != "_stdio" && role.Args[0] != "_udp-proxy") {
	301	+	return nil, fmt.Errorf("unsupported public peer command %q", role.Args)
	302	+	}
	303	+	route := peerRoute{mapping: m, command: role}
	304	+	if m.Role == "udp_peer" {
	305	+	route.pidPath = filepath.Join(runDir, "udp-peer.pid")
	306	+	factory := l.netemFactory
	307	+	if factory == nil {
	308	+	factory = newUDPNetem
	309	+	}
	310	+	netem, err := factory(udpNetemConfig{
	311	+	RTT: time.Duration(role.Transport.RTTMS) * time.Millisecond,
	312	+	LossPercent: role.Transport.LossPercent,
	313	+	TargetPath: filepath.Join(runDir, "udp-peer.target"),
	314	+	})
	315	+	if err != nil {
	316	+	return nil, fmt.Errorf("start harness UDP netem: %w", err)
	317	+	}
	318	+	route.netem = netem
	319	+	}
	320	+	shim := filepath.Join(runDir, "ssh")
	321	+	ready := filepath.Join(runDir, "udp-peer.ready")
	322	+	target := filepath.Join(runDir, "udp-peer.target")
	323	+	netemPort := 0
	324	+	if route.netem != nil {
	325	+	netemPort = route.netem.Port()
	326	+	}
	327	+	stderr := filepath.Join(runDir, "udp-peer.stderr")
	328	+	// The bootstrap protocol requires the SSH command to exit after forwarding
	329	+	// the readiness line. The long-lived proxy is owned by pidPath and receives
	330	+	// the role's unique trace identity, never the bootstrap/client identity.
	331	+	body := fmt.Sprintf(`#!/bin/sh
	332	+	set -eu
	333	+	case "$*" in
	334	+	"_stdio")
	335	+	exec env VEV_PERF_TRACE=%[1]q VEV_PERF_PROCESS_ID=%[2]q VEV_PERF_SCENARIO=%[12]q VEV_PERF_RUN=%[13]d XDG_RUNTIME_DIR=%[3]q XDG_STATE_HOME=%[4]q TERM=xterm-256color %[5]q _stdio %[6]q
	336	+	;;
	337	+	"_udp-bootstrap")
	338	+	rm -f %[7]q %[8]q %[9]q
	339	+	env VEV_PERF_TRACE=%[1]q VEV_PERF_PROCESS_ID=%[2]q VEV_PERF_SCENARIO=%[12]q VEV_PERF_RUN=%[13]d XDG_RUNTIME_DIR=%[3]q XDG_STATE_HOME=%[4]q TERM=xterm-256color %[5]q _udp-proxy %[6]q >%[7]q 2>%[8]q &
	340	+	peer=$!
	341	+	printf '%%s\n' "$peer" > %[11]q
	342	+	i=0
	343	+	while [ ! -s %[7]q ] && [ "$i" -lt 1000 ]; do sleep 0.01; i=$((i+1)); done
	344	+	test -s %[7]q
	345	+	line=$(head -n 1 %[7]q)
	346	+	printf '%%s\n' "$line" > %[9]q
	347	+	set -- $line
	348	+	test "$1" = VEV-UDP
	349	+	test -n "$3"
	350	+	printf 'VEV-UDP %%s %%s\n' %[10]d "$3"
	351	+	exit 0
	352	+	;;
	353	+	*) echo 'vev harness ssh seam rejected non-vev command' >&2; exit 64 ;;
	354	+	esac
	355	+	`, m.TracePath, m.ProcessID, runtimeDir, filepath.Join(runDir, "state"), l.bin, role.Args[1], ready, stderr, target, netemPort, route.pidPath, m.Scenario, m.Run)
	356	+	if err := os.WriteFile(shim, []byte(body), 0o700); err != nil {
	357	+	if route.netem != nil {
	358	+	_ = route.netem.Close()
	359	+	}
	360	+	return nil, err
	361	+	}
	362	+	l.mu.Lock()
	363	+	if l.peers == nil {
	364	+	l.peers = make(map[string]peerRoute)
	365	+	}
	366	+	if _, exists := l.peers[runDir]; exists {
	367	+	l.mu.Unlock()
	368	+	if route.netem != nil {
	369	+	_ = route.netem.Close()
	370	+	}
	371	+	return nil, fmt.Errorf("duplicate transport peer for %s", runDir)
	372	+	}
	373	+	l.peers[runDir] = route
	374	+	l.mu.Unlock()
	375	+	peer := preparedPeer{route: route}
	376	+	if peerOwnsRuntime {
	377	+	peer.cleanupRuntime = func() error { return l.releaseRuntime(runDir) }
	378	+	}
	379	+	return peer, nil
	380	+	}
	381	+
	382	+	func remoteMode(t transport) (string, error) {
	383	+	switch t.Kind {
	384	+	case "ssh_stdio":
	385	+	return "stdio", nil
	386	+	case "udp":
	387	+	return "udp", nil
	388	+	default:
	389	+	return "", fmt.Errorf("transport %q cannot route through a remote peer", t.ID)
	390	+	}
	391	+	}
	392	+
	393	+	// traceEnvironment is the complete process-local identity supplied by the
	394	+	// persisted harness processMapping. App turns Scenario/Run into production
	395	+	// mark correlation; sequence/request/epoch remain process-local operation IDs.
	396	+	func traceEnvironment(m processMapping) []string {
	397	+	return []string{
	398	+	"VEV_PERF_TRACE=" + m.TracePath,
	399	+	"VEV_PERF_PROCESS_ID=" + m.ProcessID,
	400	+	"VEV_PERF_SCENARIO=" + m.Scenario,
	401	+	"VEV_PERF_RUN=" + strconv.Itoa(m.Run),
	402	+	}
	403	+	}
	404	+
	405	+	func withoutEnv(env []string, names ...string) []string {
	406	+	remove := make(map[string]bool, len(names))
	407	+	for _, name := range names {
	408	+	remove[name] = true
	409	+	}
	410	+	out := make([]string, 0, len(env))
	411	+	for _, entry := range env {
	412	+	name, _, _ := strings.Cut(entry, "=")
	413	+	if !remove[name] {
	414	+	out = append(out, entry)
	415	+	}
	416	+	}
	417	+	return out
	418	+	}
	419	+
	420	+	func (p *cliProcess) copyTerminal() {
	421	+	defer close(p.chunks)
	422	+	buf := make([]byte, 4096)
	423	+	for {
	424	+	n, err := p.pty.Read(buf)
	425	+	if n > 0 {
	426	+	chunk := append([]byte(nil), buf[:n]...)
	427	+	if _, writeErr := p.output.Write(chunk); writeErr != nil {
	428	+	return
	429	+	}
	430	+	select {
	431	+	case p.chunks <- chunk:
	432	+	case <-p.done:
	433	+	return
	434	+	}
	435	+	}
	436	+	if err != nil {
	437	+	return
	438	+	}
	439	+	}
	440	+	}
	441	+
	442	+	// WaitReady confirms the explicitly launched public daemon owns its unique
	443	+	// socket before a client is launched. Without this barrier, the normal client
	444	+	// auto-spawn behavior can create an unmanifested daemon process during startup.
	445	+	func (p *cliProcess) WaitReady() error {
	446	+	if p.readyPath == "" {
	447	+	return errors.New("daemon readiness path is empty")
	448	+	}
	449	+	ticker := time.NewTicker(10 * time.Millisecond)
	450	+	defer ticker.Stop()
	451	+	timer := time.NewTimer(10 * time.Second)
	452	+	defer timer.Stop()
	453	+	for {
	454	+	if info, err := os.Stat(p.readyPath); err == nil {
	455	+	if info.Mode()&os.ModeSocket == 0 {
	456	+	return fmt.Errorf("daemon readiness path %s is not a socket", p.readyPath)
	457	+	}
	458	+	return nil
	459	+	}
	460	+	select {
	461	+	case err := <-p.waitErr:
	462	+	p.waitErr <- err
	463	+	if err == nil {
	464	+	return errors.New("daemon exited before readiness")
	465	+	}
	466	+	return fmt.Errorf("daemon exited before readiness: %w", err)
	467	+	case <-ticker.C:
	468	+	case <-timer.C:
	469	+	return fmt.Errorf("timed out waiting for daemon socket %s", p.readyPath)
	470	+	}
	471	+	}
	472	+	}
	473	+
	474	+	func (p *cliProcess) Warmup(input []byte) error {
	475	+	if p.pty == nil \|\| len(input) == 0 {
	476	+	return nil
	477	+	}
	478	+	// A client cannot write application output until it has received Welcome and
	479	+	// entered raw mode. Require that observable output before submitting any
	480	+	// warmup input, so the PTY's line discipline cannot echo it locally.
	481	+	if !p.terminalReady {
	482	+	if err := p.waitForTerminalReady(); err != nil {
	483	+	return err
	484	+	}
	485	+	p.terminalReady = true
	486	+	}
	487	+	if _, err := p.pty.Write(input); err != nil {
	488	+	return err
	489	+	}
	490	+	// Do not begin measured input until the public client has rendered a real
	491	+	// shell response to warmup input. This is intentionally not a measurement.
	492	+	return p.waitForTerminalMarker(inputMarker(input), input)
	493	+	}
	494	+
	495	+	// Measure records a boundary only after the harness-owned terminal output file
	496	+	// has accepted application output and Sync has reported a successful flush.
	497	+	// The marker is injected through the real client PTY, never fabricated by the
	498	+	// harness. PTY local echo is discarded and cannot satisfy this boundary.
	499	+	func (p *cliProcess) Measure(input []byte, injected, flushed func() error) error {
	500	+	if p.pty == nil {
	501	+	return errors.New("measured process has no terminal PTY")
	502	+	}
	503	+	if _, err := p.pty.Write(input); err != nil {
	504	+	return err
	505	+	}
	506	+	// The injected callback runs only after the owned PTY accepts the bytes.
	507	+	if err := injected(); err != nil {
	508	+	return err
	509	+	}
	510	+	if err := p.waitForTerminalMarker(inputMarker(input), input); err != nil {
	511	+	return err
	512	+	}
	513	+	if err := p.output.Sync(); err != nil {
	514	+	return fmt.Errorf("flush terminal output: %w", err)
	515	+	}
	516	+	return flushed()
	517	+	}
	518	+
	519	+	// waitForTerminalReady waits for public client output produced before any
	520	+	// harness input. The client enters raw mode before its output loop can render
	521	+	// this initial prompt/state, so this is an observable readiness boundary.
	522	+	func (p *cliProcess) waitForTerminalReady() error {
	523	+	timer := time.NewTimer(10 * time.Second)
	524	+	defer timer.Stop()
	525	+	for {
	526	+	select {
	527	+	case chunk, ok := <-p.chunks:
	528	+	if !ok {
	529	+	return p.terminalClosedError("readiness output")
	530	+	}
	531	+	if len(chunk) == 0 {
	532	+	continue
	533	+	}
	534	+	if err := p.output.Sync(); err != nil {
	535	+	return fmt.Errorf("flush terminal readiness output: %w", err)
	536	+	}
	537	+	return nil
	538	+	case err := <-p.waitErr:
	539	+	// Keep the exit result available to Close, which owns reaping.
	540	+	p.waitErr <- err
	541	+	if err == nil {
	542	+	return errors.New("client exited before readiness output")
	543	+	}
	544	+	return fmt.Errorf("client exited before readiness output: %w", err)
	545	+	case <-timer.C:
	546	+	return errors.New("timed out waiting for client readiness output")
	547	+	}
	548	+	}
	549	+	}
	550	+
	551	+	func (p *cliProcess) terminalClosedError(boundary string) error {
	552	+	if p.waitErr != nil {
	553	+	select {
	554	+	case err := <-p.waitErr:
	555	+	// Close owns reaping and may need the result after a failed boundary.
	556	+	p.waitErr <- err
	557	+	if err != nil {
	558	+	return fmt.Errorf("client terminal closed before %s: %w", boundary, err)
	559	+	}
	560	+	case <-time.After(10 * time.Millisecond):
	561	+	}
	562	+	}
	563	+	return fmt.Errorf("client terminal closed before %s", boundary)
	564	+	}
	565	+
	566	+	func (p *cliProcess) waitForTerminalMarker(marker, input []byte) error {
	567	+	var received []byte
	568	+	echo := newPTYLocalEcho(input)
	569	+	timer := time.NewTimer(10 * time.Second)
	570	+	defer timer.Stop()
	571	+	for !bytes.Contains(received, marker) {
	572	+	select {
	573	+	case chunk, ok := <-p.chunks:
	574	+	if !ok {
	575	+	return p.terminalClosedError("application output")
	576	+	}
	577	+	received = append(received, echo.applicationOutput(chunk)...)
	578	+	case err := <-p.waitErr:
	579	+	// Keep the exit result available to Close, which owns reaping.
	580	+	p.waitErr <- err
	581	+	if err == nil {
	582	+	return errors.New("client exited before application output")
	583	+	}
	584	+	return fmt.Errorf("client exited before application output: %w", err)
	585	+	case <-timer.C:
	586	+	return errors.New("timed out waiting for application terminal output")
	587	+	}
	588	+	}
	589	+	return nil
	590	+	}
	591	+
	592	+	// ptyLocalEcho removes the one ordered input echo that a not-yet-raw slave PTY
	593	+	// can produce. It normalizes NL to the conventional ONLCR echo spelling; any
	594	+	// divergence is retained as application output. The line discipline emits the
	595	+	// echo before the child can read and respond to the same write.
	596	+	type ptyLocalEcho struct {
	597	+	expected []byte
	598	+	matched int
	599	+	done bool
	600	+	}
	601	+
	602	+	func (e *ptyLocalEcho) applicationOutput(chunk []byte) []byte {
	603	+	if e.done \|\| len(e.expected) == 0 {
	604	+	return chunk
	605	+	}
	606	+	for i, b := range chunk {
	607	+	if b != e.expected[e.matched] {
	608	+	e.done = true
	609	+	return append([]byte(nil), chunk[i:]...)
	610	+	}
	611	+	e.matched++
	612	+	if e.matched == len(e.expected) {
	613	+	e.done = true
	614	+	return append([]byte(nil), chunk[i+1:]...)
	615	+	}
	616	+	}
	617	+	return nil
	618	+	}
	619	+
	620	+	func (p *cliProcess) Close() error {
	621	+	var result error
	622	+	p.closed.Do(func() {
	623	+	if p.done != nil {
	624	+	close(p.done)
	625	+	}
	626	+	// A public terminal client owns a shell session. Ask that shell to exit
	627	+	// before closing the PTY so its transport can finish its receive span;
	628	+	// this avoids turning ordinary harness teardown into an aborted trace.
	629	+	exited := false
	630	+	if p.shutdown != nil && p.waitErr != nil {
	631	+	select {
	632	+	case <-p.waitErr:
	633	+	exited = true
	634	+	default:
	635	+	// The client may have just ended the last session and raced this
	636	+	// best-effort public shutdown. The normal bounded reaping path below
	637	+	// remains authoritative in that case.
	638	+	_ = p.shutdown()
	639	+	}
	640	+	}
	641	+	if p.pty != nil && p.waitErr != nil {
	642	+	_, _ = p.pty.Write([]byte("exit\n"))
	643	+	select {
	644	+	case <-p.waitErr:
	645	+	exited = true
	646	+	case <-p.closeDeadline():
	647	+	}
	648	+	}
	649	+	if p.pty != nil {
	650	+	_ = p.pty.Close()
	651	+	}
	652	+	if p.waitErr != nil && !exited {
	653	+	select {
	654	+	case <-p.waitErr:
	655	+	case <-p.closeDeadline():
	656	+	// Teardown is deliberately bounded: a stuck public CLI or child
	657	+	// must not hold a local harness run indefinitely.
	658	+	p.forceProcessGroupCleanup()
	659	+	select {
	660	+	case <-p.waitErr:
	661	+	case <-p.closeDeadline():
	662	+	p.forceProcessGroupKill()
	663	+	select {
	664	+	case <-p.waitErr:
	665	+	case <-p.closeDeadline():
	666	+	result = errors.New("timed out reaping public CLI process")
	667	+	}
	668	+	}
	669	+	}
	670	+	}
	671	+	if p.output != nil {
	672	+	if err := p.output.Close(); result == nil {
	673	+	result = err
	674	+	}
	675	+	}
	676	+	if p.cleanupRuntime != nil {
	677	+	if err := p.cleanupRuntime(); result == nil {
	678	+	result = err
	679	+	}
	680	+	}
	681	+	})
	682	+	return result
	683	+	}
	684	+
	685	+	func (p *cliProcess) closeDeadline() <-chan time.Time {
	686	+	if p.waitTimeout != nil {
	687	+	return p.waitTimeout()
	688	+	}
	689	+	if p.shutdown != nil {
	690	+	// A public kill asks the daemon to finish bounded detached notices and
	691	+	// close blocked carriage before it exits; do not turn that graceful
	692	+	// protocol into a SIGKILL at the two-second client deadline.
	693	+	return time.After(5 * time.Second)
	694	+	}
	695	+	return time.After(2 * time.Second)
	696	+	}
	697	+
	698	+	func (p *cliProcess) forceProcessGroupCleanup() {
	699	+	if p.forceCleanup != nil {
	700	+	p.forceCleanup()
	701	+	return
	702	+	}
	703	+	if p.cmd != nil && p.cmd.Process != nil {
	704	+	// The client is a session/process-group leader; forced cleanup must
	705	+	// include its ssh seam/_stdio descendants.
	706	+	_ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGTERM)
	707	+	}
	708	+	}
	709	+
	710	+	func (p *cliProcess) forceProcessGroupKill() {
	711	+	if p.forceKill != nil {
	712	+	p.forceKill()
	713	+	return
	714	+	}
	715	+	if p.cmd != nil && p.cmd.Process != nil {
	716	+	_ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL)
	717	+	}
	718	+	}
	719	+
	720	+	// openPTY is the small Linux PTY boundary needed to drive the public terminal
	721	+	// client. No daemon or adapter implementation is imported by this command.
	722	+	func openPTY() (os.File, os.File, error) {
	723	+	masterFD, err := unix.Open("/dev/ptmx", unix.O_RDWR\|unix.O_NOCTTY, 0)
	724	+	if err != nil {
	725	+	return nil, nil, err
	726	+	}
	727	+	closeMaster := func(err error) (os.File, os.File, error) { _ = unix.Close(masterFD); return nil, nil, err }
	728	+	number, err := unix.IoctlGetInt(masterFD, unix.TIOCGPTN)
	729	+	if err != nil {
	730	+	return closeMaster(err)
	731	+	}
	732	+	if err := unix.IoctlSetPointerInt(masterFD, unix.TIOCSPTLCK, 0); err != nil {
	733	+	return closeMaster(err)
	734	+	}
	735	+	slaveFD, err := unix.Open(fmt.Sprintf("/dev/pts/%d", number), unix.O_RDWR\|unix.O_NOCTTY, 0)
	736	+	if err != nil {
	737	+	return closeMaster(err)
	738	+	}
	739	+	return os.NewFile(uintptr(masterFD), "vev-client-master"), os.NewFile(uintptr(slaveFD), "vev-client-slave"), nil
	740	+	}
	741	+
	742	+	func inputMarker(input []byte) []byte {
	743	+	start := bytes.Index(input, []byte("__VEV_HARNESS_"))
	744	+	if start < 0 {
	745	+	return input
	746	+	}
	747	+	end := bytes.Index(input[start:], []byte(`\n`))
	748	+	if end < 0 {
	749	+	return input[start:]
	750	+	}
	751	+	return input[start : start+end]
	752	+	}

A cmd/vev-perf-harness/launcher_test.go +296 -0

@@ -0,0 +1,296 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"fmt"
	6	+	"net"
	7	+	"os"
	8	+	"os/exec"
	9	+	"path/filepath"
	10	+	"strings"
	11	+	"testing"
	12	+	"time"
	13	+	)
	14	+
	15	+	func TestCLIProcessCloseIsBoundedWhenPublicRoleDoesNotExit(t *testing.T) {
	16	+	deadline := make(chan time.Time)
	17	+	close(deadline)
	18	+	var term, kill int
	19	+	p := &cliProcess{
	20	+	waitErr: make(chan error),
	21	+	waitTimeout: func() <-chan time.Time { return deadline },
	22	+	forceCleanup: func() { term++ },
	23	+	forceKill: func() { kill++ },
	24	+	}
	25	+	if err := p.Close(); err == nil {
	26	+	t.Fatal("stuck public role close succeeded")
	27	+	}
	28	+	if term != 1 \|\| kill != 1 {
	29	+	t.Fatalf("TERM/KILL calls = %d/%d, want 1/1", term, kill)
	30	+	}
	31	+	}
	32	+
	33	+	func TestCLIProcessWaitReadyRequiresDaemonSocket(t *testing.T) {
	34	+	path := filepath.Join(t.TempDir(), "daemon.sock")
	35	+	listener, err := net.Listen("unix", path)
	36	+	if err != nil {
	37	+	t.Fatal(err)
	38	+	}
	39	+	defer listener.Close()
	40	+	if err := (&cliProcess{readyPath: path, waitErr: make(chan error, 1)}).WaitReady(); err != nil {
	41	+	t.Fatal(err)
	42	+	}
	43	+	}
	44	+
	45	+	func TestCLILauncherIsolatesRoleXDGEnvironmentAcrossRepetitions(t *testing.T) {
	46	+	captureDir := t.TempDir()
	47	+	t.Setenv("CAPTURE_DIR", captureDir)
	48	+	t.Setenv("XDG_RUNTIME_DIR", filepath.Join(t.TempDir(), "inherited-runtime"))
	49	+	t.Setenv("XDG_STATE_HOME", filepath.Join(t.TempDir(), "inherited-state"))
	50	+
	51	+	bin := filepath.Join(t.TempDir(), "capture-env")
	52	+	if err := os.WriteFile(bin, []byte("#!/bin/sh\nenv > \"$CAPTURE_DIR/$VEV_PERF_PROCESS_ID\"\n"), 0o700); err != nil {
	53	+	t.Fatal(err)
	54	+	}
	55	+	launcher := &cliLauncher{bin: bin}
	56	+	for run := 1; run <= 3; run++ {
	57	+	runDir := filepath.Join(t.TempDir(), fmt.Sprintf("run-%d", run))
	58	+	if err := os.MkdirAll(runDir, 0o700); err != nil {
	59	+	t.Fatal(err)
	60	+	}
	61	+	roles := []string{"daemon", "client"}
	62	+	for _, role := range roles {
	63	+	id := fmt.Sprintf("run-%d-%s", run, role)
	64	+	p, err := launcher.Launch(processMapping{
	65	+	Role: role, TracePath: filepath.Join(runDir, role+".jsonl"), ProcessID: id,
	66	+	}, roleCommand{Args: []string{"test"}})
	67	+	if err != nil {
	68	+	t.Fatalf("run %d %s launch: %v", run, role, err)
	69	+	}
	70	+	runtimeDir := launcher.runtimes[runDir]
	71	+	if err := p.Close(); err != nil {
	72	+	t.Fatalf("run %d %s close: %v", run, role, err)
	73	+	}
	74	+
	75	+	env, err := os.ReadFile(filepath.Join(captureDir, id))
	76	+	if err != nil {
	77	+	t.Fatalf("run %d %s capture: %v", run, role, err)
	78	+	}
	79	+	for _, want := range []string{
	80	+	"XDG_RUNTIME_DIR=" + runtimeDir,
	81	+	"XDG_STATE_HOME=" + filepath.Join(runDir, "state"),
	82	+	} {
	83	+	name := strings.SplitN(want, "=", 2)[0]
	84	+	if count := bytes.Count(env, []byte(name+"=")); count != 1 {
	85	+	t.Fatalf("run %d %s %s entries = %d, want exactly one; env:\n%s", run, role, name, count, env)
	86	+	}
	87	+	if !bytes.Contains(env, []byte(want+"\n")) {
	88	+	t.Fatalf("run %d %s missing %q; env:\n%s", run, role, want, env)
	89	+	}
	90	+	}
	91	+	}
	92	+	if err := launcher.releaseRuntime(runDir); err != nil {
	93	+	t.Fatalf("run %d release runtime: %v", run, err)
	94	+	}
	95	+	}
	96	+	}
	97	+
	98	+	func TestCLIProcessCloseUsesPublicDaemonShutdown(t *testing.T) {
	99	+	wait := make(chan error, 1)
	100	+	wait <- nil
	101	+	var shutdown int
	102	+	p := &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; return nil }}
	103	+	if err := p.Close(); err != nil {
	104	+	t.Fatal(err)
	105	+	}
	106	+	if shutdown != 0 {
	107	+	t.Fatalf("shutdown calls = %d, want 0 for an exited daemon", shutdown)
	108	+	}
	109	+
	110	+	wait = make(chan error, 1)
	111	+	p = &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; wait <- nil; return nil }}
	112	+	if err := p.Close(); err != nil {
	113	+	t.Fatal(err)
	114	+	}
	115	+	if shutdown != 1 {
	116	+	t.Fatalf("shutdown calls = %d, want 1", shutdown)
	117	+	}
	118	+	}
	119	+
	120	+	func TestHarnessCanonicalLocalSmokeRealRoles(t *testing.T) {
	121	+	if testing.Short() {
	122	+	t.Skip("public CLI smoke")
	123	+	}
	124	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	125	+	if err != nil {
	126	+	t.Fatal(err)
	127	+	}
	128	+	bin := filepath.Join(t.TempDir(), "vev")
	129	+	build := exec.Command("go", "build", "-o", bin, "./")
	130	+	build.Dir = root
	131	+	if output, err := build.CombinedOutput(); err != nil {
	132	+	t.Fatalf("build public CLI: %v\n%s", err, output)
	133	+	}
	134	+	m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
	135	+	if err != nil {
	136	+	t.Fatal(err)
	137	+	}
	138	+	if err := validateManifest(m); err != nil {
	139	+	t.Fatal(err)
	140	+	}
	141	+	var local scenario
	142	+	for _, s := range m.Scenarios {
	143	+	if s.Topology == "1x4" && s.Workload == "idle" && s.Transport == "local" {
	144	+	local = s
	145	+	break
	146	+	}
	147	+	}
	148	+	if local.ID == "" {
	149	+	t.Fatal("canonical local scenario missing")
	150	+	}
	151	+	dir := t.TempDir()
	152	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	153	+	if err != nil {
	154	+	t.Fatal(err)
	155	+	}
	156	+	defer raw.Close()
	157	+	h := defaultHarness()
	158	+	h.clock = &fakeClock{} // injected duration keeps the public 30s contract bounded in test
	159	+	h.launcher = &cliLauncher{bin: bin}
	160	+	result, err := h.runOne(options{vevBin: bin, out: dir, duration: minimumDuration}, m, local, 1, raw)
	161	+	if err != nil {
	162	+	t.Fatal(err)
	163	+	}
	164	+	if result.Samples < minimumInIntervalEventSamples \|\| len(result.Processes) != 2 {
	165	+	t.Fatalf("canonical local smoke result=%+v", result)
	166	+	}
	167	+	}
	168	+
	169	+	func TestTraceEnvironmentMatchesProcessMapping(t *testing.T) {
	170	+	mapping := processMapping{TracePath: "/tmp/trace.jsonl", ProcessID: "p-01", Scenario: "1x4-idle-local", Run: 3}
	171	+	got := strings.Join(traceEnvironment(mapping), "\n")
	172	+	for _, want := range []string{"VEV_PERF_TRACE=/tmp/trace.jsonl", "VEV_PERF_PROCESS_ID=p-01", "VEV_PERF_SCENARIO=1x4-idle-local", "VEV_PERF_RUN=3"} {
	173	+	if !strings.Contains(got, want) {
	174	+	t.Errorf("trace environment missing %q: %s", want, got)
	175	+	}
	176	+	}
	177	+	}
	178	+
	179	+	func TestCLIProcessCloseGracefulDetachCompletesSpanBeforeForcedCleanup(t *testing.T) {
	180	+	var order []string
	181	+	waitErr := make(chan error, 1)
	182	+	timeout := make(chan time.Time)
	183	+	p := &cliProcess{
	184	+	pty: &orderedPTY{
	185	+	write: func(input []byte) {
	186	+	if string(input) != "exit\n" {
	187	+	t.Fatalf("graceful exit input=%q", input)
	188	+	}
	189	+	order = append(order, "exit_written")
	190	+	// The graceful client exit has closed its transport and emitted
	191	+	// its adapter receive-end mark before cmd.Wait completes.
	192	+	order = append(order, "adapter_receive_end")
	193	+	waitErr <- nil
	194	+	},
	195	+	close: func() { order = append(order, "pty_closed") },
	196	+	},
	197	+	output: &orderedOutput{close: func() { order = append(order, "output_closed") }},
	198	+	waitErr: waitErr,
	199	+	waitTimeout: func() <-chan time.Time { return timeout },
	200	+	forceCleanup: func() {
	201	+	order = append(order, "forced_process_group_cleanup")
	202	+	},
	203	+	}
	204	+
	205	+	if err := p.Close(); err != nil {
	206	+	t.Fatal(err)
	207	+	}
	208	+	if !equalStrings(order, []string{"exit_written", "adapter_receive_end", "pty_closed", "output_closed"}) {
	209	+	t.Fatalf("cleanup order=%q", order)
	210	+	}
	211	+	}
	212	+
	213	+	func TestCLIProcessWarmupWaitsForDelayedTerminalReadiness(t *testing.T) {
	214	+	allowWrite := make(chan struct{})
	215	+	pty, output := &stagedPTY{allow: allowWrite, writes: make(chan []byte, 1)}, &fakeOutput{}
	216	+	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte)}
	217	+	input := workloadInput(scenario{ID: "s"}, 1, "warmup")
	218	+	done := make(chan error, 1)
	219	+	go func() { done <- p.Warmup(input) }()
	220	+
	221	+	// This is public client output from the initial application prompt/state,
	222	+	// emitted only after the client has entered raw mode. A premature PTY write
	223	+	// blocks on allowWrite and therefore cannot consume this delayed readiness.
	224	+	go func() {
	225	+	p.chunks <- []byte("shell prompt$ ")
	226	+	close(allowWrite)
	227	+	}()
	228	+	select {
	229	+	case got := <-pty.writes:
	230	+	if !bytes.Equal(got, input) {
	231	+	t.Fatalf("warmup write=%q want %q", got, input)
	232	+	}
	233	+	case <-time.After(time.Second):
	234	+	t.Fatal("warmup did not wait for and then proceed from client readiness")
	235	+	}
	236	+	go func() { p.chunks <- append([]byte("application "), inputMarker(input)...); close(p.chunks) }()
	237	+	if err := <-done; err != nil {
	238	+	t.Fatal(err)
	239	+	}
	240	+	if output.syncs != 1 {
	241	+	t.Fatalf("readiness output was not flushed: syncs=%d", output.syncs)
	242	+	}
	243	+	}
	244	+
	245	+	func TestCLIProcessRejectsPTYLocalEchoAsFlushEvidence(t *testing.T) {
	246	+	pty, output := &fakePTY{}, &fakeOutput{}
	247	+	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte, 1)}
	248	+	input := workloadInput(scenario{ID: "s"}, 1, "measured-1")
	249	+	p.chunks <- newPTYLocalEcho(input).expected
	250	+	close(p.chunks)
	251	+	flushed := false
	252	+	if err := p.Measure(input, func() error { return nil }, func() error { flushed = true; return nil }); err == nil {
	253	+	t.Fatal("local PTY echo satisfied terminal flush boundary")
	254	+	}
	255	+	if flushed \|\| output.syncs != 0 {
	256	+	t.Fatalf("local echo stamped a flush: flushed=%t syncs=%d", flushed, output.syncs)
	257	+	}
	258	+	}
	259	+
	260	+	func TestCLIProcessPairsApplicationOutputWithSuccessfulFlush(t *testing.T) {
	261	+	pty, output := &fakePTY{}, &fakeOutput{}
	262	+	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte, 1)}
	263	+	input := workloadInput(scenario{ID: "s"}, 1, "measured-1")
	264	+	echo := newPTYLocalEcho(input).expected
	265	+	p.chunks <- append(append([]byte(nil), echo...), append([]byte("application "), inputMarker(input)...)...)
	266	+	order := []string{}
	267	+	if err := p.Measure(input, func() error { order = append(order, "injected"); return nil }, func() error { order = append(order, "flushed"); return nil }); err != nil {
	268	+	t.Fatal(err)
	269	+	}
	270	+	if len(pty.writes) != 1 \|\| output.syncs != 1 \|\| !equalStrings(order, []string{"injected", "flushed"}) {
	271	+	t.Fatalf("writes=%q syncs=%d boundary order=%q", pty.writes, output.syncs, order)
	272	+	}
	273	+	}
	274	+
	275	+	func TestHarnessUsesPublicRoleCommandsAndPTYWorkloads(t *testing.T) {
	276	+	for _, tc := range []struct {
	277	+	role string
	278	+	want []string
	279	+	}{
	280	+	{"daemon", []string{"--daemon"}},
	281	+	{"client", []string{"new", "perf-s-001"}},
	282	+	{"ssh_stdio_peer", []string{"_stdio", "perf-s-001"}},
	283	+	{"udp_peer", []string{"_udp-proxy", "perf-s-001"}},
	284	+	} {
	285	+	t.Run(tc.role, func(t *testing.T) {
	286	+	got := roleArgs(scenario{ID: "s", Transport: "local"}, processMapping{Role: tc.role, Run: 1})
	287	+	if !equalStrings(got.Args, tc.want) {
	288	+	t.Fatalf("args=%q want %q", got.Args, tc.want)
	289	+	}
	290	+	})
	291	+	}
	292	+	input := string(workloadInput(scenario{ID: "s", Workload: "interactive_flood"}, 1, "measured-1"))
	293	+	if string(inputMarker([]byte(input))) != "__VEV_HARNESS_s_r1_measured-1__" \|\| !bytes.Contains([]byte(input), []byte("while [ $i -lt 128 ]")) \|\| !strings.HasSuffix(input, "printf '__VEV_HARNESS_s_r1_measured-1__\\n'\n") {
	294	+	t.Fatalf("workload is not real PTY shell input with an observable marker: %q", input)
	295	+	}
	296	+	}

A cmd/vev-perf-harness/main.go +20 -0

@@ -0,0 +1,20 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"os"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/vev/pkg/safedir"
	9	+	)
	10	+
	11	+	func main() {
	12	+	if err := run(os.Args[1:], defaultHarness()); err != nil {
	13	+	fmt.Fprintln(os.Stderr, "vev-perf-harness:", err)
	14	+	os.Exit(2)
	15	+	}
	16	+	}
	17	+
	18	+	func defaultHarness() *harness {
	19	+	return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, createRunDir: createExclusiveRunDir, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}
	20	+	}

A cmd/vev-perf-harness/model.go +132 -0

@@ -0,0 +1,132 @@
	1	+	package main
	2	+
	3	+	import "time"
	4	+
	5	+	const (
	6	+	minimumDuration = 30 * time.Second
	7	+	minimumRepetitions = 10
	8	+	minimumInIntervalEventSamples = 10
	9	+	measuredEventCadence = time.Second
	10	+	)
	11	+
	12	+	type options struct {
	13	+	vevBin, manifest, out string
	14	+	scenario string
	15	+	warmup, duration time.Duration
	16	+	repetitions int
	17	+	}
	18	+
	19	+	type manifest struct {
	20	+	Schema uint16 `json:"schema"`
	21	+	Topologies []topology `json:"topologies"`
	22	+	Workloads []string `json:"workloads"`
	23	+	Transports []transport `json:"transports"`
	24	+	Scenarios []scenario `json:"scenarios"`
	25	+	}
	26	+
	27	+	type topology struct {
	28	+	ID string `json:"id"`
	29	+	Geometry string `json:"geometry"`
	30	+	RowsPerPane int `json:"rows_per_pane"`
	31	+	}
	32	+
	33	+	type transport struct {
	34	+	ID string `json:"id"`
	35	+	Kind string `json:"kind"`
	36	+	RTTMS int `json:"rtt_ms,omitempty"`
	37	+	LossPercent int `json:"loss_percent,omitempty"`
	38	+	}
	39	+
	40	+	type scenario struct {
	41	+	ID string `json:"id"`
	42	+	Topology string `json:"topology"`
	43	+	Workload string `json:"workload"`
	44	+	Transport string `json:"transport"`
	45	+	Roles []string `json:"roles"`
	46	+	InapplicableReason string `json:"inapplicable_reason,omitempty"`
	47	+	}
	48	+
	49	+	type processMapping struct {
	50	+	ProcessID string `json:"process_id"`
	51	+	ClockDomain string `json:"clock_domain"`
	52	+	TracePath string `json:"trace_path"`
	53	+	Role string `json:"role"`
	54	+	Scenario string `json:"scenario"`
	55	+	Run int `json:"run"`
	56	+	Identity string `json:"identity"`
	57	+	}
	58	+
	59	+	type runManifest struct {
	60	+	Scenario string `json:"scenario"`
	61	+	Run int `json:"run"`
	62	+	Processes []processMapping `json:"processes"`
	63	+	}
	64	+
	65	+	// measuredInterval belongs solely to the harness clock domain. A paired event
	66	+	// is eligible only when both owned boundaries fall inside this interval.
	67	+	type measuredInterval struct {
	68	+	Start int64
	69	+	End int64
	70	+	}
	71	+
	72	+	type eventSample struct {
	73	+	Sequence uint64 `json:"sequence"`
	74	+	Injected int64 `json:"injected_tick"`
	75	+	Flushed int64 `json:"flushed_tick"`
	76	+	Latency int64 `json:"latency_nanos"`
	77	+	}
	78	+
	79	+	type span struct {
	80	+	Component, Name string
	81	+	Samples []int64
	82	+	}
	83	+
	84	+	type runResult struct {
	85	+	Spans []span `json:"process_local_spans"`
	86	+	Scenario string `json:"scenario"`
	87	+	Run int `json:"run"`
	88	+	Samples int `json:"samples"`
	89	+	EndToEnd []int64 `json:"end_to_end_nanos"`
	90	+	Event distribution `json:"event_end_to_end"`
	91	+	Cadence distribution `json:"event_cadence_nanos"`
	92	+	CadenceSamples []int64 `json:"-"`
	93	+	MaxGap int64 `json:"event_max_gap_nanos"`
	94	+	Processes []processMapping `json:"processes"`
	95	+	}
	96	+
	97	+	type summary struct {
	98	+	Schema uint16 `json:"schema"`
	99	+	GitSHA string `json:"git_sha"`
	100	+	Warmup string `json:"warmup"`
	101	+	Duration string `json:"duration"`
	102	+	Repetitions int `json:"repetitions"`
	103	+	EndToEnd distribution `json:"harness_end_to_end"`
	104	+	Cadence distribution `json:"harness_event_cadence_nanos"`
	105	+	MaxGap int64 `json:"harness_event_max_gap_nanos"`
	106	+	RunP50Dispersion distribution `json:"run_p50_dispersion"`
	107	+	Spans []spanSummary `json:"process_local_spans"`
	108	+	Runs int `json:"runs"`
	109	+	}
	110	+
	111	+	type distribution struct {
	112	+	Count int `json:"count"`
	113	+	P50 int64 `json:"p50"`
	114	+	P95 int64 `json:"p95"`
	115	+	P99 int64 `json:"p99"`
	116	+	Max int64 `json:"max"`
	117	+	}
	118	+
	119	+	type spanSummary struct {
	120	+	Component string `json:"component"`
	121	+	Name string `json:"name"`
	122	+	Distribution distribution `json:"distribution"`
	123	+	}
	124	+
	125	+	type clock interface {
	126	+	Now() int64
	127	+	Sleep(time.Duration)
	128	+	}
	129	+
	130	+	type systemClock struct{ start time.Time }
	131	+
	132	+	var spanPairs = []spanPair{{"capture_start", "capture_end", "capture_duration"}, {"compose_start", "compose_end", "compose_duration"}, {"diff_start", "diff_end", "diff_duration"}, {"queue_enqueued", "queue_dequeued", "queue_wait"}, {"ack_blocked_start", "ack_blocked_end", "ack_blocked_interval"}, {"emit_start", "emit_end", "emit_duration"}, {"adapter_send_start", "adapter_send_end", "adapter_send_duration"}, {"adapter_receive_start", "adapter_receive_end", "adapter_receive_duration"}}

A cmd/vev-perf-harness/options_manifest.go +221 -0

@@ -0,0 +1,221 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"errors"
	6	+	"flag"
	7	+	"fmt"
	8	+	"io"
	9	+	"os"
	10	+	"path/filepath"
	11	+	)
	12	+
	13	+	func parseOptions(args []string) (options, error) {
	14	+	var o options
	15	+	fs := flag.NewFlagSet("vev-perf-harness", flag.ContinueOnError)
	16	+	fs.SetOutput(io.Discard)
	17	+	fs.StringVar(&o.vevBin, "vev-bin", "", "public vev binary")
	18	+	fs.StringVar(&o.manifest, "manifest", "", "manifest")
	19	+	fs.StringVar(&o.out, "out", "", "output directory")
	20	+	fs.StringVar(&o.scenario, "scenario", "", "one canonical scenario ID (after full manifest validation)")
	21	+	fs.DurationVar(&o.warmup, "warmup", 0, "warmup")
	22	+	fs.DurationVar(&o.duration, "duration", 0, "measurement duration")
	23	+	fs.IntVar(&o.repetitions, "repetitions", 0, "repetitions")
	24	+	if err := fs.Parse(args); err != nil {
	25	+	return o, err
	26	+	}
	27	+	seen := map[string]bool{}
	28	+	fs.Visit(func(f *flag.Flag) { seen[f.Name] = true })
	29	+	for _, name := range []string{"vev-bin", "manifest", "out", "warmup", "duration", "repetitions"} {
	30	+	if !seen[name] {
	31	+	return o, fmt.Errorf("--%s is required", name)
	32	+	}
	33	+	}
	34	+	if o.vevBin == "" \|\| o.manifest == "" \|\| o.out == "" {
	35	+	return o, errors.New("--vev-bin, --manifest, and --out are required")
	36	+	}
	37	+	if o.warmup < 0 {
	38	+	return o, errors.New("--warmup must not be negative")
	39	+	}
	40	+	if o.duration < minimumDuration {
	41	+	return o, fmt.Errorf("--duration must be at least %s", minimumDuration)
	42	+	}
	43	+	if o.repetitions < minimumRepetitions {
	44	+	return o, fmt.Errorf("--repetitions must be at least %d", minimumRepetitions)
	45	+	}
	46	+	return o, nil
	47	+	}
	48	+
	49	+	// resolvePathOptions captures the harness cwd once, before role launchers
	50	+	// assign their isolated working directories. All downstream filesystem access,
	51	+	// manifests, and process environments consequently refer to the same paths.
	52	+	func resolvePathOptions(o options) (options, error) {
	53	+	cwd, err := os.Getwd()
	54	+	if err != nil {
	55	+	return o, fmt.Errorf("resolve harness working directory: %w", err)
	56	+	}
	57	+	absolute := func(path string) string {
	58	+	if filepath.IsAbs(path) {
	59	+	return filepath.Clean(path)
	60	+	}
	61	+	return filepath.Clean(filepath.Join(cwd, path))
	62	+	}
	63	+	o.vevBin = absolute(o.vevBin)
	64	+	o.manifest = absolute(o.manifest)
	65	+	o.out = absolute(o.out)
	66	+	return o, nil
	67	+	}
	68	+
	69	+	func readManifest(path string) (manifest, error) {
	70	+	f, err := os.Open(path)
	71	+	if err != nil {
	72	+	return manifest{}, err
	73	+	}
	74	+	var m manifest
	75	+	if err := json.NewDecoder(f).Decode(&m); err != nil {
	76	+	return manifest{}, closeFile(f, err)
	77	+	}
	78	+	if err := f.Close(); err != nil {
	79	+	return manifest{}, err
	80	+	}
	81	+	return m, nil
	82	+	}
	83	+
	84	+	func validateManifest(m manifest) error {
	85	+	if m.Schema != 1 {
	86	+	return errors.New("unsupported manifest schema")
	87	+	}
	88	+	tops := map[string]bool{}
	89	+	for _, t := range m.Topologies {
	90	+	if t.ID == "" \|\| t.Geometry != "120x40" \|\| t.RowsPerPane != 10000 {
	91	+	return fmt.Errorf("invalid topology %q", t.ID)
	92	+	}
	93	+	tops[t.ID] = true
	94	+	}
	95	+	wantT := []string{"1x4", "4x1", "4x4", "8x1"}
	96	+	for _, v := range wantT {
	97	+	if !tops[v] {
	98	+	return fmt.Errorf("missing canonical topology %s", v)
	99	+	}
	100	+	}
	101	+	works := set(m.Workloads)
	102	+	for _, v := range []string{"idle", "active_output", "all_output", "inactive_output", "interactive_flood", "copy_search", "resize_sweep", "snapshot_output_resize", "attach_restore_tab_switch"} {
	103	+	if !works[v] {
	104	+	return fmt.Errorf("missing canonical workload %s", v)
	105	+	}
	106	+	}
	107	+	trans := map[string]bool{}
	108	+	for _, t := range m.Transports {
	109	+	if t.ID == "" {
	110	+	return errors.New("transport missing id")
	111	+	}
	112	+	if err := validateTransportFixture(t); err != nil {
	113	+	return err
	114	+	}
	115	+	trans[t.ID] = true
	116	+	}
	117	+	for _, v := range []string{"local", "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct"} {
	118	+	if !trans[v] {
	119	+	return fmt.Errorf("missing canonical transport %s", v)
	120	+	}
	121	+	}
	122	+	seen := map[string]bool{}
	123	+	covered := map[string]bool{}
	124	+	for _, s := range m.Scenarios {
	125	+	if s.ID == "" \|\| seen[s.ID] {
	126	+	return fmt.Errorf("duplicate or empty scenario id %q", s.ID)
	127	+	}
	128	+	seen[s.ID] = true
	129	+	if !tops[s.Topology] \|\| !works[s.Workload] \|\| !trans[s.Transport] {
	130	+	return fmt.Errorf("scenario %s references unknown matrix entry", s.ID)
	131	+	}
	132	+	if s.InapplicableReason == "" && len(s.Roles) == 0 {
	133	+	return fmt.Errorf("scenario %s has no process roles", s.ID)
	134	+	}
	135	+	if s.InapplicableReason != "" && len(s.Roles) != 0 {
	136	+	return fmt.Errorf("inapplicable scenario %s must not launch roles", s.ID)
	137	+	}
	138	+	if s.InapplicableReason == "" && !equalRoleSet(s.Roles, requiredRoles(s.Transport)) {
	139	+	return fmt.Errorf("scenario %s roles do not route transport %s", s.ID, s.Transport)
	140	+	}
	141	+	covered[s.Topology+"\x00"+s.Workload+"\x00"+s.Transport] = true
	142	+	}
	143	+	for topologyID := range tops {
	144	+	for workload := range works {
	145	+	for transportID := range trans {
	146	+	key := topologyID + "\x00" + workload + "\x00" + transportID
	147	+	if !covered[key] {
	148	+	return fmt.Errorf("missing scenario or inapplicable reason for %s/%s/%s", topologyID, workload, transportID)
	149	+	}
	150	+	}
	151	+	}
	152	+	}
	153	+	return nil
	154	+	}
	155	+
	156	+	// selectScenario is intentionally called only after validateManifest. A bounded
	157	+	// local evidence run therefore cannot mask an incomplete canonical matrix.
	158	+	func selectScenario(m manifest, id string) ([]scenario, error) {
	159	+	for _, s := range m.Scenarios {
	160	+	if s.ID != id {
	161	+	continue
	162	+	}
	163	+	if s.InapplicableReason != "" {
	164	+	return nil, fmt.Errorf("scenario %s is inapplicable: %s", id, s.InapplicableReason)
	165	+	}
	166	+	return []scenario{s}, nil
	167	+	}
	168	+	return nil, fmt.Errorf("unknown canonical scenario %s", id)
	169	+	}
	170	+
	171	+	func requiredRoles(transportID string) []string {
	172	+	switch transportID {
	173	+	case "local":
	174	+	return []string{"daemon", "client"}
	175	+	case "ssh_stdio":
	176	+	return []string{"daemon", "client", "ssh_stdio_peer"}
	177	+	default:
	178	+	return []string{"daemon", "client", "udp_peer"}
	179	+	}
	180	+	}
	181	+
	182	+	func equalRoleSet(got, want []string) bool {
	183	+	if len(got) != len(want) {
	184	+	return false
	185	+	}
	186	+	seen := make(map[string]bool, len(got))
	187	+	for _, role := range got {
	188	+	seen[role] = true
	189	+	}
	190	+	for _, role := range want {
	191	+	if !seen[role] {
	192	+	return false
	193	+	}
	194	+	}
	195	+	return true
	196	+	}
	197	+
	198	+	func validateTransportFixture(t transport) error {
	199	+	want := map[string]transport{
	200	+	"local": {ID: "local", Kind: "local"},
	201	+	"ssh_stdio": {ID: "ssh_stdio", Kind: "ssh_stdio"},
	202	+	"udp_baseline": {ID: "udp_baseline", Kind: "udp"},
	203	+	"udp_25ms": {ID: "udp_25ms", Kind: "udp", RTTMS: 25},
	204	+	"udp_100ms": {ID: "udp_100ms", Kind: "udp", RTTMS: 100},
	205	+	"udp_loss_0pct": {ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0},
	206	+	"udp_loss_1pct": {ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1},
	207	+	}
	208	+	w, ok := want[t.ID]
	209	+	if !ok \|\| t.Kind != w.Kind \|\| t.RTTMS != w.RTTMS \|\| t.LossPercent != w.LossPercent {
	210	+	return fmt.Errorf("invalid canonical transport fixture %+v", t)
	211	+	}
	212	+	return nil
	213	+	}
	214	+
	215	+	func set(v []string) map[string]bool {
	216	+	r := map[string]bool{}
	217	+	for _, x := range v {
	218	+	r[x] = true
	219	+	}
	220	+	return r
	221	+	}

A cmd/vev-perf-harness/options_manifest_test.go +201 -0

@@ -0,0 +1,201 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"errors"
	6	+	"os"
	7	+	"path/filepath"
	8	+	"strings"
	9	+	"testing"
	10	+	"time"
	11	+	)
	12	+
	13	+	func TestHarnessScenarioSelectionKeepsCanonicalManifestValidation(t *testing.T) {
	14	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	15	+	if err != nil {
	16	+	t.Fatal(err)
	17	+	}
	18	+	manifestPath := filepath.Join(root, "testdata", "perf", "manifest.json")
	19	+	m, err := readManifest(manifestPath)
	20	+	if err != nil {
	21	+	t.Fatal(err)
	22	+	}
	23	+	selected := m.Scenarios[0]
	24	+	h := defaultHarness()
	25	+	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
	26	+	out := filepath.Join(t.TempDir(), "out")
	27	+	if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
	28	+	t.Fatal(err)
	29	+	}
	30	+	var runs []runResult
	31	+	b, err := os.ReadFile(filepath.Join(out, "runs.json"))
	32	+	if err != nil {
	33	+	t.Fatal(err)
	34	+	}
	35	+	if err := json.Unmarshal(b, &runs); err != nil {
	36	+	t.Fatal(err)
	37	+	}
	38	+	if len(runs) != 10 {
	39	+	t.Fatalf("selected canonical runs=%d, want 10", len(runs))
	40	+	}
	41	+	for _, result := range runs {
	42	+	if result.Scenario != selected.ID {
	43	+	t.Fatalf("selected scenario = %q, want %q", result.Scenario, selected.ID)
	44	+	}
	45	+	}
	46	+	}
	47	+
	48	+	func TestHarnessScenarioSelectionRejectsIncompleteCanonicalManifest(t *testing.T) {
	49	+	m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
	50	+	if err != nil {
	51	+	t.Fatal(err)
	52	+	}
	53	+	selected := m.Scenarios[0]
	54	+	m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
	55	+	dir := t.TempDir()
	56	+	manifestPath := filepath.Join(dir, "manifest.json")
	57	+	if err := os.WriteFile(manifestPath, []byte(mustJSON(m)), 0o600); err != nil {
	58	+	t.Fatal(err)
	59	+	}
	60	+	launcher := &fakeLauncher{}
	61	+	h := defaultHarness()
	62	+	h.clock, h.launcher = &fakeClock{}, launcher
	63	+	out := filepath.Join(dir, "out")
	64	+	err = run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h)
	65	+	if err == nil \|\| !strings.Contains(err.Error(), "missing scenario or inapplicable reason") {
	66	+	t.Fatalf("incomplete manifest selection error = %v", err)
	67	+	}
	68	+	if len(launcher.mappings) != 0 {
	69	+	t.Fatalf("launched with incomplete canonical manifest: %+v", launcher.mappings)
	70	+	}
	71	+	if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
	72	+	t.Fatalf("output created before manifest validation: %v", statErr)
	73	+	}
	74	+	}
	75	+
	76	+	func TestResolvePathOptionsMakesRelativeAndAbsoluteInputsIdentical(t *testing.T) {
	77	+	cwd, err := os.Getwd()
	78	+	if err != nil {
	79	+	t.Fatal(err)
	80	+	}
	81	+	absolute := options{
	82	+	vevBin: filepath.Join(cwd, "vev"),
	83	+	manifest: filepath.Join(cwd, "testdata", "perf", "manifest.json"),
	84	+	out: filepath.Join(cwd, "testdata", "perf", "results"),
	85	+	}
	86	+	relative := options{
	87	+	vevBin: "vev",
	88	+	manifest: "testdata/perf/manifest.json",
	89	+	out: "testdata/perf/results",
	90	+	}
	91	+	gotRelative, err := resolvePathOptions(relative)
	92	+	if err != nil {
	93	+	t.Fatal(err)
	94	+	}
	95	+	gotAbsolute, err := resolvePathOptions(absolute)
	96	+	if err != nil {
	97	+	t.Fatal(err)
	98	+	}
	99	+	if gotRelative != gotAbsolute {
	100	+	t.Fatalf("relative paths resolved to %+v, absolute paths to %+v", gotRelative, gotAbsolute)
	101	+	}
	102	+	}
	103	+
	104	+	func TestHarnessFlagsRejectShortMeasurements(t *testing.T) {
	105	+	for _, args := range [][]string{
	106	+	{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "29s", "--repetitions", "10"},
	107	+	{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "9"},
	108	+	{"--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"},
	109	+	} {
	110	+	if _, err := parseOptions(args); err == nil {
	111	+	t.Fatalf("parseOptions(%v) succeeded", args)
	112	+	}
	113	+	}
	114	+	if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"}); err == nil {
	115	+	t.Fatal("missing --warmup accepted")
	116	+	}
	117	+	if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}); err != nil {
	118	+	t.Fatal(err)
	119	+	}
	120	+	}
	121	+
	122	+	func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
	123	+	m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
	124	+	if err != nil {
	125	+	t.Fatal(err)
	126	+	}
	127	+	if err := validateManifest(m); err != nil {
	128	+	t.Fatal(err)
	129	+	}
	130	+	if len(m.Scenarios) != 497 {
	131	+	t.Fatalf("scenarios=%d", len(m.Scenarios))
	132	+	}
	133	+	m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
	134	+	if err := validateManifest(m); err == nil {
	135	+	t.Fatal("missing combination accepted")
	136	+	}
	137	+	}
	138	+
	139	+	func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
	140	+	dir := t.TempDir()
	141	+	raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	142	+	if err != nil {
	143	+	t.Fatal(err)
	144	+	}
	145	+	defer raw.Close()
	146	+	launcher := &fakeLauncher{}
	147	+	h := defaultHarness()
	148	+	h.clock = &fakeClock{}
	149	+	h.launcher = launcher
	150	+	o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
	151	+	r, err := h.runOne(o, manifest{}, scenario{ID: "4x4-local", Roles: []string{"daemon", "client"}}, 1, raw)
	152	+	if err != nil {
	153	+	t.Fatal(err)
	154	+	}
	155	+	if !launcher.manifestPresent {
	156	+	t.Fatal("launched before complete run manifest")
	157	+	}
	158	+	if len(r.EndToEnd) < minimumInIntervalEventSamples \|\| r.Samples < minimumInIntervalEventSamples {
	159	+	t.Fatalf("samples=%+v", r)
	160	+	}
	161	+	seen := map[string]bool{}
	162	+	for _, m := range r.Processes {
	163	+	if m.ProcessID == "" \|\| m.ClockDomain != m.ProcessID \|\| m.TracePath == "" \|\| seen[m.TracePath] {
	164	+	t.Fatalf("bad mapping %+v", m)
	165	+	}
	166	+	seen[m.TracePath] = true
	167	+	if _, err := os.Stat(m.TracePath); err != nil {
	168	+	t.Fatal(err)
	169	+	}
	170	+	}
	171	+	b, err := os.ReadFile(filepath.Join(dir, "4x4-local-run-001", "manifest.json"))
	172	+	if err != nil {
	173	+	t.Fatal(err)
	174	+	}
	175	+	var got runManifest
	176	+	if err := json.Unmarshal(b, &got); err != nil {
	177	+	t.Fatal(err)
	178	+	}
	179	+	if len(got.Processes) != 2 {
	180	+	t.Fatalf("manifest=%+v", got)
	181	+	}
	182	+	if len(launcher.process) != 2 \|\| !launcher.process[0].closed \|\| len(launcher.process[0].warmups) != 0 \|\| len(launcher.process[1].warmups) != 1 \|\| len(launcher.process[1].measures) < minimumInIntervalEventSamples {
	183	+	t.Fatalf("process lifecycle not recorded: %+v", launcher.process)
	184	+	}
	185	+	}
	186	+
	187	+	func TestHarnessRejectsMissingManifestRoleTrace(t *testing.T) {
	188	+	dir := t.TempDir()
	189	+	missing := filepath.Join(dir, "daemon.jsonl")
	190	+	client := filepath.Join(dir, "client.jsonl")
	191	+	if err := os.WriteFile(client, nil, 0o600); err != nil {
	192	+	t.Fatal(err)
	193	+	}
	194	+	_, err := mergeProcessTraces([]processMapping{
	195	+	{ProcessID: "daemon", ClockDomain: "daemon", TracePath: missing, Role: "daemon", Scenario: "s", Run: 1},
	196	+	{ProcessID: "client", ClockDomain: "client", TracePath: client, Role: "client", Scenario: "s", Run: 1},
	197	+	})
	198	+	if !errors.Is(err, os.ErrNotExist) \|\| !strings.Contains(err.Error(), "daemon role") {
	199	+	t.Fatalf("missing daemon trace error = %v, want daemon role not-exist error", err)
	200	+	}
	201	+	}

A cmd/vev-perf-harness/runner.go +439 -0

@@ -0,0 +1,439 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"context"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"os"
	10	+	"os/exec"
	11	+	"path/filepath"
	12	+	"runtime/debug"
	13	+	"sort"
	14	+	"strings"
	15	+	"time"
	16	+
	17	+	"github.com/bnema/vev/pkg/safedir"
	18	+	)
	19	+
	20	+	type harnessMark struct {
	21	+	Scenario string `json:"scenario"`
	22	+	Run int `json:"run"`
	23	+	Sequence uint64 `json:"sequence"`
	24	+	Kind string `json:"kind"`
	25	+	Tick int64 `json:"tick"`
	26	+	Valid bool `json:"valid"`
	27	+	}
	28	+
	29	+	func (c systemClock) Now() int64 { return time.Since(c.start).Nanoseconds() }
	30	+
	31	+	func (systemClock) Sleep(d time.Duration) { time.Sleep(d) }
	32	+
	33	+	func newPTYLocalEcho(input []byte) *ptyLocalEcho {
	34	+	expected := make([]byte, 0, len(input))
	35	+	for _, b := range input {
	36	+	if b == '\n' {
	37	+	expected = append(expected, '\r')
	38	+	}
	39	+	expected = append(expected, b)
	40	+	}
	41	+	return &ptyLocalEcho{expected: expected}
	42	+	}
	43	+
	44	+	type harness struct {
	45	+	clock clock
	46	+	launcher launcher
	47	+	mkdir func(string) error
	48	+	createRunDir func(string) error
	49	+	create func(string) (*os.File, error)
	50	+	removeAll func(string) error
	51	+	gitSHA func() string
	52	+	selectScenarios func(manifest) []scenario // test-only bounded fixture seam
	53	+	}
	54	+
	55	+	// createExclusiveRunDir prevents stale runtime, socket, state, or trace files
	56	+	// from an earlier invocation being mistaken for the next repetition's evidence.
	57	+	func createExclusiveRunDir(path string) error {
	58	+	if err := os.Mkdir(path, 0o700); err != nil {
	59	+	return err
	60	+	}
	61	+	if err := safedir.EnsurePrivate(path); err != nil {
	62	+	return errors.Join(err, os.Remove(path))
	63	+	}
	64	+	return nil
	65	+	}
	66	+
	67	+	// recordedGitSHA prefers build provenance so an installed public harness does
	68	+	// not depend on its launch directory being a checkout. The git fallback keeps
	69	+	// go run evidence attributable as well.
	70	+	const gitSHACommandTimeout = time.Second
	71	+
	72	+	func recordedGitSHA() string {
	73	+	if info, ok := debug.ReadBuildInfo(); ok {
	74	+	for _, setting := range info.Settings {
	75	+	if setting.Key == "vcs.revision" && setting.Value != "" {
	76	+	return setting.Value
	77	+	}
	78	+	}
	79	+	}
	80	+	ctx, cancel := context.WithTimeout(context.Background(), gitSHACommandTimeout)
	81	+	defer cancel()
	82	+	out, err := exec.CommandContext(ctx, "git", "rev-parse", "HEAD").Output()
	83	+	if err == nil && strings.TrimSpace(string(out)) != "" {
	84	+	return strings.TrimSpace(string(out))
	85	+	}
	86	+	return "unknown"
	87	+	}
	88	+
	89	+	func run(args []string, h *harness) (err error) {
	90	+	opt, err := parseOptions(args)
	91	+	if err != nil {
	92	+	return err
	93	+	}
	94	+	opt, err = resolvePathOptions(opt)
	95	+	if err != nil {
	96	+	return err
	97	+	}
	98	+	m, err := readManifest(opt.manifest)
	99	+	if err != nil {
	100	+	return err
	101	+	}
	102	+	if err := validateManifest(m); err != nil {
	103	+	return err
	104	+	}
	105	+	if err = h.mkdir(opt.out); err != nil {
	106	+	return err
	107	+	}
	108	+	raw, err := os.OpenFile(filepath.Join(opt.out, "raw-harness.jsonl"), os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
	109	+	if err != nil {
	110	+	return err
	111	+	}
	112	+	defer func() {
	113	+	if closeErr := raw.Close(); closeErr != nil {
	114	+	err = errors.Join(err, closeErr)
	115	+	}
	116	+	}()
	117	+	results := make([]runResult, 0, len(m.Scenarios)*opt.repetitions)
	118	+	var localSpans []span
	119	+	scenarios := m.Scenarios
	120	+	if opt.scenario != "" {
	121	+	scenarios, err = selectScenario(m, opt.scenario)
	122	+	if err != nil {
	123	+	return err
	124	+	}
	125	+	}
	126	+	if h.selectScenarios != nil {
	127	+	scenarios = h.selectScenarios(m)
	128	+	}
	129	+	for _, s := range scenarios {
	130	+	if s.InapplicableReason != "" {
	131	+	continue
	132	+	}
	133	+	for r := 1; r <= opt.repetitions; r++ {
	134	+	rr, err := h.runOne(opt, m, s, r, raw)
	135	+	if err != nil {
	136	+	return fmt.Errorf("scenario %s run %d: %w", s.ID, r, err)
	137	+	}
	138	+	results = append(results, rr)
	139	+	localSpans = append(localSpans, rr.Spans...)
	140	+	}
	141	+	}
	142	+	all, cadence, maxGap, runP50s := aggregateEventResults(results)
	143	+	if len(all) == 0 {
	144	+	return errors.New("no measured harness input/flush pairs")
	145	+	}
	146	+	if err := writeJSON(filepath.Join(opt.out, "runs.json"), results); err != nil {
	147	+	return err
	148	+	}
	149	+	return writeJSON(filepath.Join(opt.out, "summary.json"), summary{Schema: 1, GitSHA: h.gitSHA(), Warmup: opt.warmup.String(), Duration: opt.duration.String(), Repetitions: opt.repetitions, EndToEnd: percentiles(all), Cadence: percentiles(cadence), MaxGap: maxGap, RunP50Dispersion: percentiles(runP50s), Spans: summarizeSpans(localSpans), Runs: len(results)})
	150	+	}
	151	+
	152	+	// aggregateEventResults deliberately preserves raw event latencies for
	153	+	// end-to-end percentiles. Per-run p50s are a separate dispersion series.
	154	+	func aggregateEventResults(results []runResult) (all, cadence []int64, maxGap int64, runP50s []int64) {
	155	+	for _, result := range results {
	156	+	all = append(all, result.EndToEnd...)
	157	+	cadence = append(cadence, result.CadenceSamples...)
	158	+	runP50s = append(runP50s, result.Event.P50)
	159	+	if result.MaxGap > maxGap {
	160	+	maxGap = result.MaxGap
	161	+	}
	162	+	}
	163	+	return all, cadence, maxGap, runP50s
	164	+	}
	165	+
	166	+	func (h *harness) runOne(o options, mat manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {
	167	+	dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))
	168	+	createRunDir := h.createRunDir
	169	+	if createRunDir == nil {
	170	+	createRunDir = createExclusiveRunDir
	171	+	}
	172	+	if err = createRunDir(dir); err != nil {
	173	+	return res, fmt.Errorf("create isolated run directory: %w", err)
	174	+	}
	175	+	success := false
	176	+	defer func() {
	177	+	if !success {
	178	+	if cleanupErr := h.removeAll(dir); cleanupErr != nil {
	179	+	err = errors.Join(err, fmt.Errorf("remove failed run directory: %w", cleanupErr))
	180	+	}
	181	+	}
	182	+	}()
	183	+	maps := make([]processMapping, 0, len(s.Roles))
	184	+	for i, role := range s.Roles {
	185	+	pid := fmt.Sprintf("%s-r%03d-%s-%02d", safeName(s.ID), run, safeName(role), i+1)
	186	+	path := filepath.Join(dir, pid+".jsonl")
	187	+	f, e := h.create(path)
	188	+	if e != nil {
	189	+	return res, fmt.Errorf("allocate exclusive trace %s: %w", role, e)
	190	+	}
	191	+	if e = f.Close(); e != nil {
	192	+	return res, e
	193	+	}
	194	+	maps = append(maps, processMapping{ProcessID: pid, ClockDomain: pid, TracePath: path, Role: role, Scenario: s.ID, Run: run, Identity: "harness-assigned:" + pid})
	195	+	}
	196	+	// Persist the complete mapping before any process can be launched. This is
	197	+	// both crash evidence and the only legal source for later ID correlation.
	198	+	if err = writeJSON(filepath.Join(dir, "manifest.json"), runManifest{Scenario: s.ID, Run: run, Processes: maps}); err != nil {
	199	+	return res, err
	200	+	}
	201	+	if h.launcher == nil {
	202	+	h.launcher = &cliLauncher{bin: o.vevBin}
	203	+	}
	204	+	processes := make([]launchedRole, 0, len(maps))
	205	+	closed := false
	206	+	closeRoles := func() error {
	207	+	if closed {
	208	+	return nil
	209	+	}
	210	+	closed = true
	211	+	// Clients own SSH-stdio descendants; close them before the deferred UDP
	212	+	// peer and daemon so no role survives a failed or completed run.
	213	+	return closeLaunchedRoles(processes)
	214	+	}
	215	+	defer func() {
	216	+	if cleanupErr := closeRoles(); cleanupErr != nil {
	217	+	if err == nil {
	218	+	err = cleanupErr
	219	+	} else {
	220	+	// Keep the primary workload failure first while retaining every
	221	+	// cleanup failure in reverse launch order.
	222	+	err = errors.Join(err, cleanupErr)
	223	+	}
	224	+	}
	225	+	}()
	226	+	selectedTransport, e := manifestTransport(mat, s.Transport)
	227	+	if e != nil {
	228	+	return res, e
	229	+	}
	230	+	for _, pm := range launchOrder(maps) {
	231	+	p, e := h.launcher.Launch(pm, routeRoleArgs(s, pm, selectedTransport))
	232	+	if e != nil {
	233	+	return res, e
	234	+	}
	235	+	processes = append(processes, launchedRole{mapping: pm, process: p})
	236	+	if pm.Role == "daemon" {
	237	+	if ready, ok := p.(interface{ WaitReady() error }); ok {
	238	+	if err = ready.WaitReady(); err != nil {
	239	+	return res, err
	240	+	}
	241	+	}
	242	+	}
	243	+	}
	244	+	marks := []harnessMark{}
	245	+	recordMark := func(mark harnessMark) error {
	246	+	marks = append(marks, mark)
	247	+	if err := json.NewEncoder(raw).Encode(mark); err != nil {
	248	+	return fmt.Errorf("write raw harness mark: %w", err)
	249	+	}
	250	+	return nil
	251	+	}
	252	+	seq := uint64(0)
	253	+	for _, p := range processes {
	254	+	if p.mapping.Role != "client" {
	255	+	continue
	256	+	}
	257	+	seq++
	258	+	sequence := seq
	259	+	if err = recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "input_injected", Tick: h.clock.Now(), Valid: true}); err != nil {
	260	+	return res, err
	261	+	}
	262	+	warmupErr := p.process.Warmup(workloadInput(s, run, "warmup"))
	263	+	if markErr := recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "terminal_flushed", Tick: h.clock.Now(), Valid: warmupErr == nil}); markErr != nil {
	264	+	return res, errors.Join(warmupErr, markErr)
	265	+	}
	266	+	if warmupErr != nil {
	267	+	return res, warmupErr
	268	+	}
	269	+	}
	270	+	// The harness clock, rather than a process clock, owns the explicit warmup
	271	+	// and measured intervals. Raw evidence retains every boundary, while only
	272	+	// complete valid pairs within [Start, End) contribute measurement samples.
	273	+	h.clock.Sleep(o.warmup)
	274	+	interval := measuredInterval{Start: h.clock.Now()}
	275	+	interval.End = interval.Start + o.duration.Nanoseconds()
	276	+	nextInjection := interval.Start
	277	+	for {
	278	+	now := h.clock.Now()
	279	+	if now < nextInjection {
	280	+	h.clock.Sleep(time.Duration(nextInjection - now))
	281	+	now = h.clock.Now()
	282	+	}
	283	+	if now >= interval.End {
	284	+	break
	285	+	}
	286	+	for _, p := range processes {
	287	+	if p.mapping.Role != "client" {
	288	+	continue
	289	+	}
	290	+	seq++
	291	+	sequence := seq
	292	+	injectedRecorded := false
	293	+	flushedRecorded := false
	294	+	injected := func() error {
	295	+	if markErr := recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "input_injected", Tick: h.clock.Now(), Valid: true}); markErr != nil {
	296	+	return markErr
	297	+	}
	298	+	injectedRecorded = true
	299	+	return nil
	300	+	}
	301	+	flushed := func() error {
	302	+	if markErr := recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "terminal_flushed", Tick: h.clock.Now(), Valid: true}); markErr != nil {
	303	+	return markErr
	304	+	}
	305	+	flushedRecorded = true
	306	+	return nil
	307	+	}
	308	+	if measureErr := p.process.Measure(workloadInput(s, run, fmt.Sprintf("measured-%d", sequence)), injected, flushed); measureErr != nil {
	309	+	if injectedRecorded && !flushedRecorded {
	310	+	diagnosticErr := recordMark(harnessMark{Scenario: s.ID, Run: run, Sequence: sequence, Kind: "terminal_flushed", Tick: h.clock.Now(), Valid: false})
	311	+	return res, errors.Join(measureErr, diagnosticErr)
	312	+	}
	313	+	return res, measureErr
	314	+	}
	315	+	}
	316	+	// Cap injection rate rather than bursting events after a slow terminal
	317	+	// flush. Any resulting spacing is retained as raw cadence evidence.
	318	+	nextInjection = h.clock.Now() + measuredEventCadence.Nanoseconds()
	319	+	}
	320	+	if now := h.clock.Now(); now < interval.End {
	321	+	h.clock.Sleep(time.Duration(interval.End - now))
	322	+	}
	323	+	events, e := measuredEventSamples(marks, interval)
	324	+	if e != nil {
	325	+	return res, e
	326	+	}
	327	+	if err := requireMinimumEventSamples(events); err != nil {
	328	+	return res, err
	329	+	}
	330	+	samples := eventLatencies(events)
	331	+	cadenceSamples, maxGap := eventCadenceSamples(events)
	332	+	// Process teardown can unblock a receive and cause its failed end mark to
	333	+	// be serialized. Merge only the complete, post-cleanup per-process traces.
	334	+	if cleanupErr := closeRoles(); cleanupErr != nil {
	335	+	return res, cleanupErr
	336	+	}
	337	+	spans, e := mergeProcessTraces(maps)
	338	+	if e != nil {
	339	+	return res, e
	340	+	}
	341	+	success = true
	342	+	return runResult{Spans: spans, Scenario: s.ID, Run: run, Samples: len(samples), EndToEnd: samples, Event: percentiles(samples), Cadence: percentiles(cadenceSamples), CadenceSamples: cadenceSamples, MaxGap: maxGap, Processes: maps}, nil
	343	+	}
	344	+
	345	+	func roleArgs(s scenario, m processMapping) roleCommand {
	346	+	return routeRoleArgs(s, m, scenarioTransport(s))
	347	+	}
	348	+
	349	+	func routeRoleArgs(s scenario, m processMapping, selected transport) roleCommand {
	350	+	session := "perf-" + safeName(s.ID) + fmt.Sprintf("-%03d", m.Run)
	351	+	remote := []string{"attach", "harness@127.0.0.1:" + session}
	352	+	switch m.Role {
	353	+	case "daemon":
	354	+	return roleCommand{Args: []string{"--daemon"}}
	355	+	case "client":
	356	+	switch s.Transport {
	357	+	case "local":
	358	+	return roleCommand{Args: []string{"new", session}}
	359	+	case "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct":
	360	+	return roleCommand{Args: remote}
	361	+	}
	362	+	case "ssh_stdio_peer":
	363	+	return roleCommand{Args: []string{"_stdio", session}, Transport: transport{ID: s.Transport, Kind: "ssh_stdio"}}
	364	+	case "udp_peer":
	365	+	return roleCommand{Args: []string{"_udp-proxy", session}, Transport: selected}
	366	+	}
	367	+	return roleCommand{}
	368	+	}
	369	+
	370	+	func manifestTransport(m manifest, id string) (transport, error) {
	371	+	for _, t := range m.Transports {
	372	+	if t.ID == id {
	373	+	return t, nil
	374	+	}
	375	+	}
	376	+	// Focused unit tests may call runOne with a minimal manifest; production
	377	+	// already validates every scenario reference before this point.
	378	+	if m.Schema == 0 {
	379	+	return scenarioTransport(scenario{Transport: id}), nil
	380	+	}
	381	+	return transport{}, fmt.Errorf("scenario references missing transport %q", id)
	382	+	}
	383	+
	384	+	func scenarioTransport(s scenario) transport {
	385	+	switch s.Transport {
	386	+	case "udp_baseline":
	387	+	return transport{ID: s.Transport, Kind: "udp"}
	388	+	case "udp_25ms":
	389	+	return transport{ID: s.Transport, Kind: "udp", RTTMS: 25}
	390	+	case "udp_100ms":
	391	+	return transport{ID: s.Transport, Kind: "udp", RTTMS: 100}
	392	+	case "udp_loss_0pct":
	393	+	return transport{ID: s.Transport, Kind: "udp", LossPercent: 0}
	394	+	case "udp_loss_1pct":
	395	+	return transport{ID: s.Transport, Kind: "udp", LossPercent: 1}
	396	+	default:
	397	+	return transport{ID: s.Transport}
	398	+	}
	399	+	}
	400	+
	401	+	func launchOrder(mappings []processMapping) []processMapping {
	402	+	out := append([]processMapping(nil), mappings...)
	403	+	priority := func(role string) int {
	404	+	switch role {
	405	+	case "daemon":
	406	+	return 0
	407	+	case "ssh_stdio_peer", "udp_peer":
	408	+	return 1
	409	+	case "client":
	410	+	return 2
	411	+	default:
	412	+	return 3
	413	+	}
	414	+	}
	415	+	sort.SliceStable(out, func(i, j int) bool { return priority(out[i].Role) < priority(out[j].Role) })
	416	+	return out
	417	+	}
	418	+
	419	+	// workloadInput is shell input delivered through the client PTY. It avoids any
	420	+	// private daemon API and makes every topology/workload/transport manifest row
	421	+	// execute a real terminal workload. The marker is the output boundary awaited
	422	+	// by cliProcess; it is not a synthetic timestamp.
	423	+	func workloadInput(s scenario, run int, phase string) []byte {
	424	+	marker := fmt.Sprintf("__VEV_HARNESS_%s_r%d_%s__", safeName(s.ID), run, safeName(phase))
	425	+	body := "printf 'vev perf %s\\n'"
	426	+	switch s.Workload {
	427	+	case "active_output", "all_output", "inactive_output", "interactive_flood":
	428	+	body = "i=0; while [ $i -lt 128 ]; do printf 'vev perf output %s\\n'; i=$((i+1)); done"
	429	+	case "resize_sweep":
	430	+	body = "printf 'vev perf resize-sweep %s\\n'"
	431	+	case "copy_search":
	432	+	body = "printf 'vev perf copy-search %s\\n'"
	433	+	case "snapshot_output_resize":
	434	+	body = "printf 'vev perf snapshot-output-resize %s\\n'"
	435	+	case "attach_restore_tab_switch":
	436	+	body = "printf 'vev perf attach-restore-tab-switch %s\\n'"
	437	+	}
	438	+	return []byte(fmt.Sprintf(body+"; printf '%s\\n'\n", s.ID, marker))
	439	+	}

A cmd/vev-perf-harness/runner_test.go +456 -0

@@ -0,0 +1,456 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"os"
	9	+	"os/exec"
	10	+	"path/filepath"
	11	+	"strings"
	12	+	"testing"
	13	+	"time"
	14	+	)
	15	+
	16	+	func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
	17	+	if testing.Short() {
	18	+	t.Skip("public CLI repetition lifecycle")
	19	+	}
	20	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	21	+	if err != nil {
	22	+	t.Fatal(err)
	23	+	}
	24	+	bin := filepath.Join(t.TempDir(), "vev")
	25	+	build := exec.Command("go", "build", "-o", bin, "./")
	26	+	build.Dir = root
	27	+	if output, err := build.CombinedOutput(); err != nil {
	28	+	t.Fatalf("build public CLI: %v\n%s", err, output)
	29	+	}
	30	+	m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
	31	+	if err != nil {
	32	+	t.Fatal(err)
	33	+	}
	34	+	var local scenario
	35	+	for _, candidate := range m.Scenarios {
	36	+	if candidate.ID == "1x4-idle-local" {
	37	+	local = candidate
	38	+	break
	39	+	}
	40	+	}
	41	+	if local.ID == "" {
	42	+	t.Fatal("canonical local scenario missing")
	43	+	}
	44	+	out, err := os.MkdirTemp("", "vev-ob-")
	45	+	if err != nil {
	46	+	t.Fatal(err)
	47	+	}
	48	+	removeTestTree(t, out)
	49	+	raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	50	+	if err != nil {
	51	+	t.Fatal(err)
	52	+	}
	53	+	closeTestFile(t, raw)
	54	+	h := defaultHarness()
	55	+	h.clock = &fakeClock{}
	56	+	h.launcher = &cliLauncher{bin: bin}
	57	+	for run := 1; run <= 3; run++ {
	58	+	result, err := h.runOne(options{vevBin: bin, out: out, duration: minimumDuration}, m, local, run, raw)
	59	+	if err != nil {
	60	+	t.Fatalf("run %d: %v", run, err)
	61	+	}
	62	+	if len(result.Processes) != 2 {
	63	+	t.Fatalf("run %d process mappings = %+v", run, result.Processes)
	64	+	}
	65	+	runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
	66	+	manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
	67	+	if err != nil {
	68	+	t.Fatalf("run %d manifest: %v", run, err)
	69	+	}
	70	+	var persisted runManifest
	71	+	if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
	72	+	t.Fatalf("run %d manifest JSON: %v", run, err)
	73	+	}
	74	+	if persisted.Run != run \|\| len(persisted.Processes) != len(result.Processes) {
	75	+	t.Fatalf("run %d manifest = %+v", run, persisted)
	76	+	}
	77	+	for _, process := range result.Processes {
	78	+	if process.Role != "daemon" && process.Role != "client" {
	79	+	t.Fatalf("run %d unexpected role %q", run, process.Role)
	80	+	}
	81	+	if filepath.Dir(process.TracePath) != runDir {
	82	+	t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
	83	+	}
	84	+	trace, err := os.ReadFile(process.TracePath)
	85	+	if err != nil {
	86	+	t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
	87	+	}
	88	+	if len(bytes.TrimSpace(trace)) == 0 {
	89	+	t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
	90	+	}
	91	+	for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
	92	+	var record traceRecord
	93	+	if err := json.Unmarshal(line, &record); err != nil {
	94	+	t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
	95	+	}
	96	+	if record.ProcessID != process.ProcessID \|\| record.Scenario != local.ID \|\| record.Run != uint64(run) {
	97	+	t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
	98	+	}
	99	+	}
	100	+	}
	101	+	if _, err := os.Stat(filepath.Join(runDir, "runtime", "vev", "daemon.sock")); !errors.Is(err, os.ErrNotExist) {
	102	+	t.Fatalf("run %d left daemon socket for the next repetition: %v", run, err)
	103	+	}
	104	+	}
	105	+	}
	106	+
	107	+	func TestHarnessResolvesCanonicalRelativePathsBeforeRoleCWDChanges(t *testing.T) {
	108	+	if testing.Short() {
	109	+	t.Skip("public CLI relative-path lifecycle")
	110	+	}
	111	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	112	+	if err != nil {
	113	+	t.Fatal(err)
	114	+	}
	115	+	// Keep the real Unix-socket path below its platform limit while testing the
	116	+	// exact documented relative output hierarchy.
	117	+	workspace, err := os.MkdirTemp("", "vev-rel-")
	118	+	if err != nil {
	119	+	t.Fatal(err)
	120	+	}
	121	+	removeTestTree(t, workspace)
	122	+	bin := filepath.Join(workspace, "vev")
	123	+	build := exec.Command("go", "build", "-o", bin, "./")
	124	+	build.Dir = root
	125	+	if output, err := build.CombinedOutput(); err != nil {
	126	+	t.Fatalf("build public CLI: %v\n%s", err, output)
	127	+	}
	128	+	manifestPath := filepath.Join(workspace, "testdata", "perf", "manifest.json")
	129	+	if err := os.MkdirAll(filepath.Dir(manifestPath), 0o700); err != nil {
	130	+	t.Fatal(err)
	131	+	}
	132	+	contents, err := os.ReadFile(filepath.Join(root, "testdata", "perf", "manifest.json"))
	133	+	if err != nil {
	134	+	t.Fatal(err)
	135	+	}
	136	+	if err := os.WriteFile(manifestPath, contents, 0o600); err != nil {
	137	+	t.Fatal(err)
	138	+	}
	139	+	originalWD, err := os.Getwd()
	140	+	if err != nil {
	141	+	t.Fatal(err)
	142	+	}
	143	+	if err := os.Chdir(workspace); err != nil {
	144	+	t.Fatal(err)
	145	+	}
	146	+	t.Cleanup(func() {
	147	+	if err := os.Chdir(originalWD); err != nil {
	148	+	t.Errorf("restore cwd: %v", err)
	149	+	}
	150	+	})
	151	+
	152	+	// These are the documented relative locations. cliLauncher changes every
	153	+	// role to its run directory, so a failure to resolve them before launch
	154	+	// would make VEV_PERF_TRACE point at a nonexistent nested directory.
	155	+	h := defaultHarness()
	156	+	h.clock = &fakeClock{}
	157	+	if err := run([]string{
	158	+	"--vev-bin", "./vev",
	159	+	"--manifest", "testdata/perf/manifest.json",
	160	+	"--out", "testdata/perf/results",
	161	+	"--scenario", "1x4-idle-local",
	162	+	"--warmup", "0s",
	163	+	"--duration", "30s",
	164	+	"--repetitions", "10",
	165	+	}, h); err != nil {
	166	+	t.Fatal(err)
	167	+	}
	168	+
	169	+	out := filepath.Join(workspace, "testdata", "perf", "results")
	170	+	for run := 1; run <= 10; run++ {
	171	+	runDir := filepath.Join(out, fmt.Sprintf("1x4-idle-local-run-%03d", run))
	172	+	manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
	173	+	if err != nil {
	174	+	t.Fatalf("run %d manifest: %v", run, err)
	175	+	}
	176	+	var persisted runManifest
	177	+	if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
	178	+	t.Fatalf("run %d manifest JSON: %v", run, err)
	179	+	}
	180	+	if persisted.Run != run \|\| len(persisted.Processes) != 2 {
	181	+	t.Fatalf("run %d manifest=%+v", run, persisted)
	182	+	}
	183	+	for _, process := range persisted.Processes {
	184	+	if !filepath.IsAbs(process.TracePath) \|\| filepath.Dir(process.TracePath) != runDir {
	185	+	t.Fatalf("run %d %s trace path=%q, want absolute path in %q", run, process.Role, process.TracePath, runDir)
	186	+	}
	187	+	trace, err := os.ReadFile(process.TracePath)
	188	+	if err != nil \|\| len(bytes.TrimSpace(trace)) == 0 {
	189	+	t.Fatalf("run %d %s trace=%q err=%v", run, process.Role, process.TracePath, err)
	190	+	}
	191	+	}
	192	+	}
	193	+	}
	194	+
	195	+	func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
	196	+	out := t.TempDir()
	197	+	raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	198	+	if err != nil {
	199	+	t.Fatal(err)
	200	+	}
	201	+	closeTestFile(t, raw)
	202	+	s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
	203	+	stale := filepath.Join(out, "isolated-run-001")
	204	+	if err := os.Mkdir(stale, 0o700); err != nil {
	205	+	t.Fatal(err)
	206	+	}
	207	+	if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
	208	+	t.Fatal(err)
	209	+	}
	210	+	launcher := &fakeLauncher{}
	211	+	h := defaultHarness()
	212	+	h.clock, h.launcher = &fakeClock{}, launcher
	213	+	_, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
	214	+	if err == nil \|\| !strings.Contains(err.Error(), "create isolated run directory") {
	215	+	t.Fatalf("existing run directory error = %v", err)
	216	+	}
	217	+	if len(launcher.mappings) != 0 {
	218	+	t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
	219	+	}
	220	+	if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil \|\| string(contents) != "stale" {
	221	+	t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
	222	+	}
	223	+	}
	224	+
	225	+	func TestHarnessDoesNotInheritNestedSession(t *testing.T) {
	226	+	env := withoutEnv([]string{"VEV=session=old", "PATH=/bin", "VEV_PERF_TRACE=old"}, "VEV")
	227	+	if !equalStrings(env, []string{"PATH=/bin", "VEV_PERF_TRACE=old"}) {
	228	+	t.Fatalf("environment=%q", env)
	229	+	}
	230	+	}
	231	+
	232	+	func TestHarnessRoutesEveryRemoteFixtureThroughItsDeclaredPeer(t *testing.T) {
	233	+	cases := []struct {
	234	+	name string
	235	+	transport transport
	236	+	peer string
	237	+	wantRTT int
	238	+	wantLoss int
	239	+	}{
	240	+	{"ssh", transport{ID: "ssh_stdio", Kind: "ssh_stdio"}, "ssh_stdio_peer", 0, 0},
	241	+	{"udp baseline", transport{ID: "udp_baseline", Kind: "udp"}, "udp_peer", 0, 0},
	242	+	{"udp 25ms", transport{ID: "udp_25ms", Kind: "udp", RTTMS: 25}, "udp_peer", 25, 0},
	243	+	{"udp 100ms", transport{ID: "udp_100ms", Kind: "udp", RTTMS: 100}, "udp_peer", 100, 0},
	244	+	{"udp loss zero", transport{ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0}, "udp_peer", 0, 0},
	245	+	{"udp loss", transport{ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1}, "udp_peer", 0, 1},
	246	+	}
	247	+	for _, tc := range cases {
	248	+	t.Run(tc.name, func(t *testing.T) {
	249	+	s := scenario{ID: "route", Transport: tc.transport.ID}
	250	+	client := routeRoleArgs(s, processMapping{Role: "client", Run: 2}, tc.transport)
	251	+	peer := routeRoleArgs(s, processMapping{Role: tc.peer, Run: 2}, tc.transport)
	252	+	if !equalStrings(client.Args, []string{"attach", "harness@127.0.0.1:perf-route-002"}) {
	253	+	t.Fatalf("client did not use public remote attach: %q", client.Args)
	254	+	}
	255	+	if tc.peer == "ssh_stdio_peer" && !equalStrings(peer.Args, []string{"_stdio", "perf-route-002"}) {
	256	+	t.Fatalf("ssh peer command=%q", peer.Args)
	257	+	}
	258	+	if tc.peer == "udp_peer" && !equalStrings(peer.Args, []string{"_udp-proxy", "perf-route-002"}) {
	259	+	t.Fatalf("udp peer command=%q", peer.Args)
	260	+	}
	261	+	if peer.Transport.RTTMS != tc.wantRTT \|\| peer.Transport.LossPercent != tc.wantLoss {
	262	+	t.Fatalf("peer lost manifest network settings: %+v", peer.Transport)
	263	+	}
	264	+	})
	265	+	}
	266	+	}
	267	+
	268	+	func TestHarnessFakeRunnerRoutesClientToPeerAndCleansEveryRole(t *testing.T) {
	269	+	for _, tc := range []struct {
	270	+	name, transport, peer, peerCommand string
	271	+	}{
	272	+	{"ssh", "ssh_stdio", "ssh_stdio_peer", "_stdio"},
	273	+	{"udp baseline", "udp_baseline", "udp_peer", "_udp-proxy"},
	274	+	{"udp 25ms", "udp_25ms", "udp_peer", "_udp-proxy"},
	275	+	{"udp 100ms", "udp_100ms", "udp_peer", "_udp-proxy"},
	276	+	{"udp loss zero", "udp_loss_0pct", "udp_peer", "_udp-proxy"},
	277	+	{"udp loss", "udp_loss_1pct", "udp_peer", "_udp-proxy"},
	278	+	} {
	279	+	t.Run(tc.name, func(t *testing.T) {
	280	+	dir := t.TempDir()
	281	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	282	+	if err != nil {
	283	+	t.Fatal(err)
	284	+	}
	285	+	closeTestFile(t, raw)
	286	+	l := &fakeLauncher{}
	287	+	h := defaultHarness()
	288	+	h.clock, h.launcher = &fakeClock{}, l
	289	+	s := scenario{ID: "routed", Transport: tc.transport, Roles: []string{"daemon", "client", tc.peer}}
	290	+	if _, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, s, 1, raw); err != nil {
	291	+	t.Fatal(err)
	292	+	}
	293	+	if len(l.mappings) != 3 \|\| l.mappings[1].Role != tc.peer \|\| l.mappings[2].Role != "client" {
	294	+	t.Fatalf("dependency launch order=%+v", l.mappings)
	295	+	}
	296	+	if l.commands[1].Args[0] != tc.peerCommand \|\| l.commands[2].Args[0] != "attach" {
	297	+	t.Fatalf("commands do not connect client through declared peer: %+v", l.commands)
	298	+	}
	299	+	for i, p := range l.process {
	300	+	if !p.closed {
	301	+	t.Fatalf("role %s was not cleaned up", l.mappings[i].Role)
	302	+	}
	303	+	}
	304	+	})
	305	+	}
	306	+	}
	307	+
	308	+	func TestHarnessAggregatesRawEventsSeparatelyFromRunDispersion(t *testing.T) {
	309	+	first := append([]int64(nil), make([]int64, minimumInIntervalEventSamples+5)...)
	310	+	for i := range first {
	311	+	first[i] = 1
	312	+	}
	313	+	second := []int64{100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1_000, 1_000}
	314	+	all, _, _, runP50s := aggregateEventResults([]runResult{
	315	+	{EndToEnd: first, Event: distribution{P50: 1}},
	316	+	{EndToEnd: second, Event: distribution{P50: 100}},
	317	+	})
	318	+	if got, dispersion := percentiles(all), percentiles(runP50s); got.Count != 30 \|\| got.P99 != 1_000 \|\| dispersion.Count != 2 \|\| dispersion.P99 != 1 {
	319	+	t.Fatalf("event aggregation=%+v run dispersion=%+v", got, dispersion)
	320	+	}
	321	+	}
	322	+
	323	+	func TestHarnessRawRecordsContainEveryMarkWhileSamplesRemainFiltered(t *testing.T) {
	324	+	dir := t.TempDir()
	325	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	326	+	if err != nil {
	327	+	t.Fatal(err)
	328	+	}
	329	+	h := defaultHarness()
	330	+	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
	331	+	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "raw-complete", Roles: []string{"daemon", "client"}}, 1, raw)
	332	+	if err != nil {
	333	+	t.Fatal(err)
	334	+	}
	335	+	if err := raw.Close(); err != nil {
	336	+	t.Fatal(err)
	337	+	}
	338	+	marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
	339	+	if got, want := len(marks), 2*(result.Samples+1); got != want {
	340	+	t.Fatalf("raw marks=%d, want warmup plus every measured boundary=%d", got, want)
	341	+	}
	342	+	for i, mark := range marks[:2] {
	343	+	if mark.Sequence != 1 \|\| !mark.Valid \|\| []string{"input_injected", "terminal_flushed"}[i] != mark.Kind {
	344	+	t.Fatalf("warmup mark %d=%+v", i, mark)
	345	+	}
	346	+	}
	347	+	for sequence := uint64(2); sequence <= uint64(result.Samples+1); sequence++ {
	348	+	if marks[2(sequence-1)].Sequence != sequence \|\| marks[2(sequence-1)+1].Sequence != sequence {
	349	+	t.Fatalf("raw marks did not retain both boundaries for sequence %d: %+v", sequence, marks)
	350	+	}
	351	+	}
	352	+	}
	353	+
	354	+	func TestHarnessPersistsFailedFlushDiagnostics(t *testing.T) {
	355	+	dir := t.TempDir()
	356	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	357	+	if err != nil {
	358	+	t.Fatal(err)
	359	+	}
	360	+	failure := errors.New("flush failed")
	361	+	h := defaultHarness()
	362	+	h.clock, h.launcher = &fakeClock{}, failedFlushLauncher{err: failure}
	363	+	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "failed-flush", Roles: []string{"daemon", "client"}}, 1, raw)
	364	+	if !errors.Is(err, failure) {
	365	+	t.Fatalf("run error=%v, want failed flush", err)
	366	+	}
	367	+	if err := raw.Close(); err != nil {
	368	+	t.Fatal(err)
	369	+	}
	370	+	marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
	371	+	if got, want := len(marks), 4; got != want {
	372	+	t.Fatalf("raw marks=%d, want warmup and failed measured pair=%d: %+v", got, want, marks)
	373	+	}
	374	+	failed := marks[len(marks)-1]
	375	+	if failed.Sequence != 2 \|\| failed.Kind != "terminal_flushed" \|\| failed.Valid {
	376	+	t.Fatalf("failed flush diagnostic=%+v", failed)
	377	+	}
	378	+	}
	379	+
	380	+	func TestHarnessSchedulesMeasuredEventsAcrossFullInterval(t *testing.T) {
	381	+	dir := t.TempDir()
	382	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	383	+	if err != nil {
	384	+	t.Fatal(err)
	385	+	}
	386	+	closeTestFile(t, raw)
	387	+	h := defaultHarness()
	388	+	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
	389	+	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "schedule", Roles: []string{"daemon", "client"}}, 1, raw)
	390	+	if err != nil {
	391	+	t.Fatal(err)
	392	+	}
	393	+	if result.Samples < minimumInIntervalEventSamples \|\| result.Cadence.Count != result.Samples-1 \|\| result.MaxGap <= 0 {
	394	+	t.Fatalf("result did not include repeated measured events: %+v", result)
	395	+	}
	396	+	if err := raw.Close(); err != nil {
	397	+	t.Fatal(err)
	398	+	}
	399	+	records, err := readJSONL(filepath.Join(dir, "raw.jsonl"))
	400	+	if err != nil {
	401	+	t.Fatal(err)
	402	+	}
	403	+	if want := result.Samples*2 + 2; len(records) != want {
	404	+	t.Fatalf("raw records=%d, want %d warmup and measured boundaries", len(records), want)
	405	+	}
	406	+	}
	407	+
	408	+	func TestHarnessPreservesWorkloadAndCleanupErrors(t *testing.T) {
	409	+	dir := t.TempDir()
	410	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	411	+	if err != nil {
	412	+	t.Fatal(err)
	413	+	}
	414	+	closeTestFile(t, raw)
	415	+	primary := errors.New("primary workload failure")
	416	+	cleanup := errors.New("daemon cleanup failure")
	417	+	launcher := &fakeLauncher{
	418	+	measureErr: map[string]error{"client": primary},
	419	+	closeErr: map[string]error{"daemon": cleanup},
	420	+	}
	421	+	h := defaultHarness()
	422	+	h.clock, h.launcher = &fakeClock{}, launcher
	423	+	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "errors", Roles: []string{"daemon", "client"}}, 1, raw)
	424	+	if !errors.Is(err, primary) \|\| !errors.Is(err, cleanup) {
	425	+	t.Fatalf("workload and cleanup errors were not both preserved: %v", err)
	426	+	}
	427	+	if got, want := err.Error(), primary.Error()+"\nclose daemon role: "+cleanup.Error(); got != want {
	428	+	t.Fatalf("error ordering=%q want %q", got, want)
	429	+	}
	430	+	for i, p := range launcher.process {
	431	+	if !p.closed {
	432	+	t.Fatalf("role %s was not closed", launcher.mappings[i].Role)
	433	+	}
	434	+	}
	435	+	}
	436	+
	437	+	func TestHarnessCleansRunDirectoryOnProcessFailure(t *testing.T) {
	438	+	dir := t.TempDir()
	439	+	raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	440	+	if err != nil {
	441	+	t.Fatal(err)
	442	+	}
	443	+	closeTestFile(t, raw)
	444	+	launcher := &fakeLauncher{}
	445	+	h := defaultHarness()
	446	+	h.clock = &fakeClock{}
	447	+	// The client is the second role and fails after the daemon has started.
	448	+	h.launcher = &failingLauncher{fakeLauncher: launcher}
	449	+	o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
	450	+	if _, err := h.runOne(o, manifest{}, scenario{ID: "cleanup", Roles: []string{"daemon", "client"}}, 1, raw); err == nil {
	451	+	t.Fatal("failed process accepted")
	452	+	}
	453	+	if _, err := os.Stat(filepath.Join(dir, "cleanup-run-001")); !os.IsNotExist(err) {
	454	+	t.Fatalf("failed run directory remains: %v", err)
	455	+	}
	456	+	}

A cmd/vev-perf-harness/test_helpers_test.go +341 -0

@@ -0,0 +1,341 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"os"
	9	+	"path/filepath"
	10	+	"testing"
	11	+	"time"
	12	+	)
	13	+
	14	+	type fakeClock struct{ tick int64 }
	15	+
	16	+	func (c *fakeClock) Now() int64 { c.tick += 10; return c.tick }
	17	+
	18	+	func (c *fakeClock) Sleep(d time.Duration) {
	19	+	c.tick += d.Nanoseconds()
	20	+	}
	21	+
	22	+	type fakeUDPNetem struct {
	23	+	port int
	24	+	closed bool
	25	+	}
	26	+
	27	+	func (n *fakeUDPNetem) Port() int { return n.port }
	28	+
	29	+	func (n *fakeUDPNetem) Close() error { n.closed = true; return nil }
	30	+
	31	+	type fakeProcess struct {
	32	+	fail bool
	33	+	measureErr error
	34	+	closeErr error
	35	+	warmups [][]byte
	36	+	measures [][]byte
	37	+	closed bool
	38	+	}
	39	+
	40	+	func (p *fakeProcess) Warmup(input []byte) error {
	41	+	p.warmups = append(p.warmups, append([]byte(nil), input...))
	42	+	return nil
	43	+	}
	44	+
	45	+	func (p *fakeProcess) Measure(input []byte, injected, flush func() error) error {
	46	+	p.measures = append(p.measures, append([]byte(nil), input...))
	47	+	if p.measureErr != nil {
	48	+	return p.measureErr
	49	+	}
	50	+	if p.fail {
	51	+	return errors.New("measure failure")
	52	+	}
	53	+	if err := injected(); err != nil {
	54	+	return err
	55	+	}
	56	+	return flush()
	57	+	}
	58	+
	59	+	func (p *fakeProcess) Close() error { p.closed = true; return p.closeErr }
	60	+
	61	+	type fakeLauncher struct {
	62	+	mappings []processMapping
	63	+	args [][]string
	64	+	commands []roleCommand
	65	+	process []*fakeProcess
	66	+	measureErr map[string]error
	67	+	closeErr map[string]error
	68	+	manifestPresent bool
	69	+	}
	70	+
	71	+	func (l *fakeLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	72	+	l.mappings = append(l.mappings, m)
	73	+	l.args = append(l.args, append([]string(nil), command.Args...))
	74	+	l.commands = append(l.commands, command)
	75	+	_, e := os.Stat(filepath.Join(filepath.Dir(m.TracePath), "manifest.json"))
	76	+	l.manifestPresent = e == nil
	77	+	p := &fakeProcess{measureErr: l.measureErr[m.Role], closeErr: l.closeErr[m.Role]}
	78	+	l.process = append(l.process, p)
	79	+	return p, nil
	80	+	}
	81	+
	82	+	type lateFlushProcess struct{ clock *fakeClock }
	83	+
	84	+	func (*lateFlushProcess) Warmup([]byte) error { return nil }
	85	+
	86	+	func (p *lateFlushProcess) Measure(_ []byte, injected, flushed func() error) error {
	87	+	if err := injected(); err != nil {
	88	+	return err
	89	+	}
	90	+	p.clock.tick += minimumDuration.Nanoseconds()
	91	+	return flushed()
	92	+	}
	93	+
	94	+	func (*lateFlushProcess) Close() error { return nil }
	95	+
	96	+	type insufficientEventLauncher struct{ clock *fakeClock }
	97	+
	98	+	func (l insufficientEventLauncher) Launch(m processMapping, _ roleCommand) (launchedProcess, error) {
	99	+	if m.Role == "client" {
	100	+	return &lateFlushProcess{clock: l.clock}, nil
	101	+	}
	102	+	return &fakeProcess{}, nil
	103	+	}
	104	+
	105	+	type failedFlushProcess struct{ err error }
	106	+
	107	+	func (*failedFlushProcess) Warmup([]byte) error { return nil }
	108	+
	109	+	func (p *failedFlushProcess) Measure(_ []byte, injected, _ func() error) error {
	110	+	if err := injected(); err != nil {
	111	+	return err
	112	+	}
	113	+	return p.err
	114	+	}
	115	+
	116	+	func (*failedFlushProcess) Close() error { return nil }
	117	+
	118	+	type failedFlushLauncher struct{ err error }
	119	+
	120	+	func (l failedFlushLauncher) Launch(m processMapping, _ roleCommand) (launchedProcess, error) {
	121	+	if m.Role == "client" {
	122	+	return &failedFlushProcess{err: l.err}, nil
	123	+	}
	124	+	return &fakeProcess{}, nil
	125	+	}
	126	+
	127	+	func closeTestFile(t testing.T, f os.File) {
	128	+	t.Helper()
	129	+	t.Cleanup(func() {
	130	+	if err := f.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
	131	+	t.Error(err)
	132	+	}
	133	+	})
	134	+	}
	135	+
	136	+	func removeTestTree(t *testing.T, path string) {
	137	+	t.Helper()
	138	+	t.Cleanup(func() {
	139	+	if err := os.RemoveAll(path); err != nil {
	140	+	t.Error(err)
	141	+	}
	142	+	})
	143	+	}
	144	+
	145	+	func readHarnessMarks(t *testing.T, path string) []harnessMark {
	146	+	t.Helper()
	147	+	records, err := readJSONL(path)
	148	+	if err != nil {
	149	+	t.Fatal(err)
	150	+	}
	151	+	marks := make([]harnessMark, len(records))
	152	+	for i, record := range records {
	153	+	encoded, err := json.Marshal(record)
	154	+	if err != nil {
	155	+	t.Fatal(err)
	156	+	}
	157	+	if err := json.Unmarshal(encoded, &marks[i]); err != nil {
	158	+	t.Fatal(err)
	159	+	}
	160	+	}
	161	+	return marks
	162	+	}
	163	+
	164	+	type fakePTY struct{ writes [][]byte }
	165	+
	166	+	func (p *fakePTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
	167	+
	168	+	func (p *fakePTY) Write(b []byte) (int, error) {
	169	+	p.writes = append(p.writes, append([]byte(nil), b...))
	170	+	return len(b), nil
	171	+	}
	172	+
	173	+	func (*fakePTY) Close() error { return nil }
	174	+
	175	+	type stagedPTY struct {
	176	+	allow <-chan struct{}
	177	+	writes chan []byte
	178	+	}
	179	+
	180	+	func (*stagedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
	181	+
	182	+	func (p *stagedPTY) Write(b []byte) (int, error) {
	183	+	if p.allow != nil {
	184	+	<-p.allow
	185	+	}
	186	+	p.writes <- append([]byte(nil), b...)
	187	+	return len(b), nil
	188	+	}
	189	+
	190	+	func (*stagedPTY) Close() error { return nil }
	191	+
	192	+	type fakeOutput struct {
	193	+	bytes.Buffer
	194	+	syncs int
	195	+	}
	196	+
	197	+	func (o *fakeOutput) Sync() error { o.syncs++; return nil }
	198	+
	199	+	func (*fakeOutput) Close() error { return nil }
	200	+
	201	+	type orderedPTY struct {
	202	+	write func([]byte)
	203	+	close func()
	204	+	}
	205	+
	206	+	func (*orderedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
	207	+
	208	+	func (p *orderedPTY) Write(b []byte) (int, error) {
	209	+	if p.write != nil {
	210	+	p.write(b)
	211	+	}
	212	+	return len(b), nil
	213	+	}
	214	+
	215	+	func (p *orderedPTY) Close() error {
	216	+	p.close()
	217	+	return nil
	218	+	}
	219	+
	220	+	type orderedOutput struct{ close func() }
	221	+
	222	+	func (*orderedOutput) Write(b []byte) (int, error) { return len(b), nil }
	223	+
	224	+	func (*orderedOutput) Sync() error { return nil }
	225	+
	226	+	func (o *orderedOutput) Close() error {
	227	+	o.close()
	228	+	return nil
	229	+	}
	230	+
	231	+	type failingLauncher struct{ *fakeLauncher }
	232	+
	233	+	func (l *failingLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	234	+	p, err := l.fakeLauncher.Launch(m, command)
	235	+	if err != nil {
	236	+	return nil, err
	237	+	}
	238	+	if m.Role == "client" {
	239	+	fake, ok := p.(*fakeProcess)
	240	+	if !ok {
	241	+	return nil, fmt.Errorf("fake launcher returned %T, want *fakeProcess", p)
	242	+	}
	243	+	fake.fail = true
	244	+	}
	245	+	return p, nil
	246	+	}
	247	+
	248	+	func equalStrings(a, b []string) bool {
	249	+	if len(a) != len(b) {
	250	+	return false
	251	+	}
	252	+	for i := range a {
	253	+	if a[i] != b[i] {
	254	+	return false
	255	+	}
	256	+	}
	257	+	return true
	258	+	}
	259	+
	260	+	type traceLauncher struct{ fakeLauncher }
	261	+
	262	+	func (l *traceLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	263	+	p, err := l.fakeLauncher.Launch(m, command)
	264	+	if err != nil {
	265	+	return nil, err
	266	+	}
	267	+	f, err := os.OpenFile(m.TracePath, os.O_WRONLY\|os.O_APPEND, 0o600)
	268	+	if err != nil {
	269	+	return nil, err
	270	+	}
	271	+	for i, pair := range spanPairs {
	272	+	for n, kind := range []string{pair.start, pair.end} {
	273	+	b, err := json.Marshal(traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: uint64(i + 1), RequestID: 1, Epoch: 1, Kind: kind, Tick: int64(i20 + n5), Valid: true})
	274	+	if err != nil {
	275	+	return nil, errors.Join(err, f.Close())
	276	+	}
	277	+	if _, err := f.Write(append(b, '\n')); err != nil {
	278	+	return nil, errors.Join(err, f.Close())
	279	+	}
	280	+	}
	281	+	}
	282	+	if err := f.Close(); err != nil {
	283	+	return nil, err
	284	+	}
	285	+	return p, nil
	286	+	}
	287	+
	288	+	type closeTraceProcess struct {
	289	+	*fakeProcess
	290	+	traceEnd func() error
	291	+	}
	292	+
	293	+	func (p *closeTraceProcess) Close() error {
	294	+	p.closed = true
	295	+	return errors.Join(p.closeErr, p.traceEnd())
	296	+	}
	297	+
	298	+	type closeTraceLauncher struct{ fakeLauncher }
	299	+
	300	+	func (l *closeTraceLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	301	+	p, err := l.fakeLauncher.Launch(m, command)
	302	+	if err != nil {
	303	+	return nil, err
	304	+	}
	305	+	start := traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: 1, RequestID: 1, Epoch: 1, Kind: "adapter_receive_start", Tick: 10, Valid: true}
	306	+	if err := appendTraceRecord(m.TracePath, start); err != nil {
	307	+	return nil, err
	308	+	}
	309	+	fake, ok := p.(*fakeProcess)
	310	+	if !ok {
	311	+	return nil, fmt.Errorf("fake launcher returned %T, want *fakeProcess", p)
	312	+	}
	313	+	return &closeTraceProcess{fakeProcess: fake, traceEnd: func() error {
	314	+	return appendTraceRecord(m.TracePath, traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: 1, RequestID: 1, Epoch: 1, Kind: "adapter_receive_end", Tick: 20, Valid: true})
	315	+	}}, nil
	316	+	}
	317	+
	318	+	func appendTraceRecord(path string, record traceRecord) error {
	319	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_APPEND, 0o600)
	320	+	if err != nil {
	321	+	return err
	322	+	}
	323	+	b, err := json.Marshal(record)
	324	+	if err != nil {
	325	+	return errors.Join(err, f.Close())
	326	+	}
	327	+	if _, err := f.Write(append(b, '\n')); err != nil {
	328	+	return errors.Join(err, f.Close())
	329	+	}
	330	+	return f.Close()
	331	+	}
	332	+
	333	+	// boundedLocalConcurrentTrace is an excerpt from the second run of the
	334	+	// bounded local public-CLI smoke. Receive sequence 3 is in flight while the
	335	+	// same process serializes sequence 4; shared-process records therefore cannot
	336	+	// have a global sequence ordering requirement.
	337	+	const boundedLocalConcurrentTrace = `{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":3,"request_id":3,"epoch":3,"kind":"adapter_receive_start","tick":1783902621495284782,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	338	+	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":4,"request_id":4,"epoch":4,"kind":"adapter_send_start","tick":1783902621495364552,"bytes":96,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	339	+	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":4,"request_id":4,"epoch":4,"kind":"adapter_send_end","tick":1783902621495386452,"bytes":96,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	340	+	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":3,"request_id":3,"epoch":3,"kind":"adapter_receive_end","tick":1783902621495463332,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	341	+	`

A cmd/vev-perf-harness/udp_netem.go +271 -0

@@ -0,0 +1,271 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"container/heap"
	5	+	"errors"
	6	+	"fmt"
	7	+	"net"
	8	+	"os"
	9	+	"strconv"
	10	+	"strings"
	11	+	"sync"
	12	+	"sync/atomic"
	13	+	"time"
	14	+	)
	15	+
	16	+	// udpNetem is a harness-owned, user-space UDP relay. It sits between the
	17	+	// public dgram client and vev's public _udp-proxy readiness address, so fixture
	18	+	// impairment is real network carriage rather than an ignored vev option.
	19	+	type udpNetem interface {
	20	+	Port() int
	21	+	Close() error
	22	+	}
	23	+
	24	+	type udpNetemFactory func(udpNetemConfig) (udpNetem, error)
	25	+
	26	+	type udpNetemConfig struct {
	27	+	// RTT is split equally across both relay directions.
	28	+	RTT time.Duration
	29	+	LossPercent int
	30	+	TargetPath string
	31	+	}
	32	+
	33	+	// udpNetemQueueCapacity bounds both packets waiting to be scheduled and
	34	+	// packets waiting for their delivery deadline. When it is full, the relay
	35	+	// drops the arriving packet and records it in queueOverflowDrops.
	36	+	const udpNetemQueueCapacity = 256
	37	+
	38	+	type scheduledPacket struct {
	39	+	packet []byte
	40	+	to net.Addr
	41	+	due time.Time
	42	+	}
	43	+
	44	+	type scheduledPackets []scheduledPacket
	45	+
	46	+	func (p scheduledPackets) Len() int { return len(p) }
	47	+	func (p scheduledPackets) Less(i, j int) bool { return p[i].due.Before(p[j].due) }
	48	+	func (p scheduledPackets) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
	49	+	func (p *scheduledPackets) Push(x any) {
	50	+	packet, ok := x.(scheduledPacket)
	51	+	if !ok {
	52	+	panic(fmt.Sprintf("scheduled packet type %T", x))
	53	+	}
	54	+	p = append(p, packet)
	55	+	}
	56	+	func (p *scheduledPackets) Pop() any {
	57	+	old := *p
	58	+	n := len(old)
	59	+	item := old[n-1]
	60	+	*p = old[:n-1]
	61	+	return item
	62	+	}
	63	+
	64	+	type udpNetemRelay struct {
	65	+	config udpNetemConfig
	66	+	conn net.PacketConn
	67	+	done chan struct{}
	68	+	wg sync.WaitGroup
	69	+	once sync.Once
	70	+	closeErr error
	71	+
	72	+	// slots bounds queued and deadline-pending packets together. The reader and
	73	+	// the sole scheduler are the relay's only goroutines.
	74	+	slots chan struct{}
	75	+	queue chan scheduledPacket
	76	+	queueOverflowDrops atomic.Uint64
	77	+	deliveryErrors atomic.Uint64
	78	+
	79	+	mu sync.Mutex
	80	+	client, dst net.Addr
	81	+	packet uint64
	82	+	}
	83	+
	84	+	func newUDPNetem(config udpNetemConfig) (udpNetem, error) {
	85	+	if config.RTT < 0 \|\| config.LossPercent < 0 \|\| config.LossPercent > 100 \|\| config.TargetPath == "" {
	86	+	return nil, fmt.Errorf("invalid UDP netem fixture: rtt=%s loss=%d target=%q", config.RTT, config.LossPercent, config.TargetPath)
	87	+	}
	88	+	conn, err := net.ListenPacket("udp", "127.0.0.1:0")
	89	+	if err != nil {
	90	+	return nil, err
	91	+	}
	92	+	r := &udpNetemRelay{
	93	+	config: config,
	94	+	conn: conn,
	95	+	done: make(chan struct{}),
	96	+	slots: make(chan struct{}, udpNetemQueueCapacity),
	97	+	queue: make(chan scheduledPacket, udpNetemQueueCapacity),
	98	+	}
	99	+	r.wg.Add(2)
	100	+	go r.run()
	101	+	go r.schedule()
	102	+	return r, nil
	103	+	}
	104	+
	105	+	func (r *udpNetemRelay) Port() int {
	106	+	addr, ok := r.conn.LocalAddr().(*net.UDPAddr)
	107	+	if !ok {
	108	+	return 0
	109	+	}
	110	+	return addr.Port
	111	+	}
	112	+
	113	+	func (r *udpNetemRelay) Close() error {
	114	+	r.once.Do(func() {
	115	+	close(r.done)
	116	+	r.closeErr = r.conn.Close()
	117	+	r.wg.Wait()
	118	+	if drops := r.queueOverflowDrops.Load(); drops != 0 {
	119	+	r.closeErr = errors.Join(r.closeErr, fmt.Errorf("udp netem queue overflow drops: %d", drops))
	120	+	}
	121	+	if writes := r.deliveryErrors.Load(); writes != 0 {
	122	+	r.closeErr = errors.Join(r.closeErr, fmt.Errorf("udp netem delivery errors: %d", writes))
	123	+	}
	124	+	})
	125	+	return r.closeErr
	126	+	}
	127	+
	128	+	func (r *udpNetemRelay) run() {
	129	+	defer r.wg.Done()
	130	+	buf := make([]byte, 64*1024)
	131	+	for {
	132	+	n, from, err := r.conn.ReadFrom(buf)
	133	+	if err != nil {
	134	+	return
	135	+	}
	136	+	to, ok := r.destination(from)
	137	+	if !ok \|\| r.drop() {
	138	+	continue
	139	+	}
	140	+	// Copy because PacketConn reuses buf on its next read.
	141	+	r.enqueue(scheduledPacket{packet: append([]byte(nil), buf[:n]...), to: to, due: time.Now().Add(r.config.RTT / 2)})
	142	+	}
	143	+	}
	144	+
	145	+	// enqueue implements the explicit overflow policy: never block the reader;
	146	+	// drop a packet if the bounded scheduler capacity has been reached.
	147	+	func (r *udpNetemRelay) enqueue(packet scheduledPacket) bool {
	148	+	select {
	149	+	case r.slots <- struct{}{}:
	150	+	case <-r.done:
	151	+	return false
	152	+	default:
	153	+	r.queueOverflowDrops.Add(1)
	154	+	return false
	155	+	}
	156	+	select {
	157	+	case r.queue <- packet:
	158	+	return true
	159	+	case <-r.done:
	160	+	<-r.slots
	161	+	return false
	162	+	}
	163	+	}
	164	+
	165	+	func (r *udpNetemRelay) schedule() {
	166	+	defer r.wg.Done()
	167	+	pending := scheduledPackets{}
	168	+	heap.Init(&pending)
	169	+	timer := time.NewTimer(time.Hour)
	170	+	if !timer.Stop() {
	171	+	<-timer.C
	172	+	}
	173	+	defer timer.Stop()
	174	+	var timerC <-chan time.Time
	175	+	resetTimer := func() {
	176	+	if timerC != nil && !timer.Stop() {
	177	+	select {
	178	+	case <-timer.C:
	179	+	default:
	180	+	}
	181	+	}
	182	+	if pending.Len() == 0 {
	183	+	timerC = nil
	184	+	return
	185	+	}
	186	+	delay := time.Until(pending[0].due)
	187	+	if delay < 0 {
	188	+	delay = 0
	189	+	}
	190	+	timer.Reset(delay)
	191	+	timerC = timer.C
	192	+	}
	193	+	for {
	194	+	resetTimer()
	195	+	select {
	196	+	case <-r.done:
	197	+	return
	198	+	case packet := <-r.queue:
	199	+	heap.Push(&pending, packet)
	200	+	case <-timerC:
	201	+	now := time.Now()
	202	+	for pending.Len() > 0 && !pending[0].due.After(now) {
	203	+	packet, ok := heap.Pop(&pending).(scheduledPacket)
	204	+	if !ok {
	205	+	panic("scheduled packet heap contained unexpected type")
	206	+	}
	207	+	select {
	208	+	case <-r.done:
	209	+	return
	210	+	default:
	211	+	if _, err := r.conn.WriteTo(packet.packet, packet.to); err != nil {
	212	+	select {
	213	+	case <-r.done:
	214	+	default:
	215	+	r.deliveryErrors.Add(1)
	216	+	}
	217	+	}
	218	+	<-r.slots
	219	+	}
	220	+	}
	221	+	}
	222	+	}
	223	+	}
	224	+
	225	+	func (r *udpNetemRelay) destination(from net.Addr) (net.Addr, bool) {
	226	+	r.mu.Lock()
	227	+	defer r.mu.Unlock()
	228	+	if r.dst == nil {
	229	+	dst, err := udpNetemTarget(r.config.TargetPath)
	230	+	if err != nil {
	231	+	return nil, false
	232	+	}
	233	+	r.dst = dst
	234	+	}
	235	+	if sameUDPAddress(from, r.dst) {
	236	+	return r.client, r.client != nil
	237	+	}
	238	+	r.client = from
	239	+	return r.dst, true
	240	+	}
	241	+
	242	+	func (r *udpNetemRelay) drop() bool {
	243	+	r.mu.Lock()
	244	+	defer r.mu.Unlock()
	245	+	r.packet++
	246	+	// The deterministic cycle makes the configured percentage auditable: a 1%
	247	+	// fixture drops exactly one carriage packet in every 100 attempts.
	248	+	return r.config.LossPercent > 0 && int((r.packet-1)%100) < r.config.LossPercent
	249	+	}
	250	+
	251	+	func udpNetemTarget(path string) (net.Addr, error) {
	252	+	line, err := os.ReadFile(path)
	253	+	if err != nil {
	254	+	return nil, err
	255	+	}
	256	+	fields := strings.Fields(string(line))
	257	+	if len(fields) != 3 \|\| fields[0] != "VEV-UDP" {
	258	+	return nil, errors.New("invalid UDP proxy readiness")
	259	+	}
	260	+	port, err := strconv.Atoi(fields[1])
	261	+	if err != nil \|\| port < 1 \|\| port > 65535 \|\| fields[2] == "" {
	262	+	return nil, errors.New("invalid UDP proxy readiness")
	263	+	}
	264	+	return &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: port}, nil
	265	+	}
	266	+
	267	+	func sameUDPAddress(a, b net.Addr) bool {
	268	+	ua, aOK := a.(*net.UDPAddr)
	269	+	ub, bOK := b.(*net.UDPAddr)
	270	+	return aOK && bOK && ua.Port == ub.Port && ua.IP.Equal(ub.IP)
	271	+	}

A cmd/vev-perf-harness/udp_netem_test.go +147 -0

@@ -0,0 +1,147 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"net"
	5	+	"os"
	6	+	"path/filepath"
	7	+	"strconv"
	8	+	"testing"
	9	+	"time"
	10	+	)
	11	+
	12	+	func closeUDPFixture(t *testing.T, c interface{ Close() error }) {
	13	+	t.Helper()
	14	+	t.Cleanup(func() {
	15	+	if err := c.Close(); err != nil {
	16	+	t.Error(err)
	17	+	}
	18	+	})
	19	+	}
	20	+
	21	+	func TestUDPNetemDropsWhenBoundedSchedulerIsFull(t *testing.T) {
	22	+	r := &udpNetemRelay{
	23	+	done: make(chan struct{}),
	24	+	slots: make(chan struct{}, udpNetemQueueCapacity),
	25	+	queue: make(chan scheduledPacket, udpNetemQueueCapacity),
	26	+	}
	27	+	packet := scheduledPacket{packet: []byte("bounded"), to: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 1}}
	28	+	for range udpNetemQueueCapacity {
	29	+	if !r.enqueue(packet) {
	30	+	t.Fatal("enqueue() dropped packet before scheduler capacity was full")
	31	+	}
	32	+	}
	33	+	if r.enqueue(packet) {
	34	+	t.Fatal("enqueue() accepted packet beyond scheduler capacity")
	35	+	}
	36	+	if got := r.queueOverflowDrops.Load(); got != 1 {
	37	+	t.Fatalf("queue overflow drops=%d, want 1", got)
	38	+	}
	39	+	}
	40	+
	41	+	func TestUDPNetemCloseRejectsQueueOverflow(t *testing.T) {
	42	+	netem, err := newUDPNetem(udpNetemConfig{TargetPath: filepath.Join(t.TempDir(), "target")})
	43	+	if err != nil {
	44	+	t.Fatal(err)
	45	+	}
	46	+	relay, ok := netem.(*udpNetemRelay)
	47	+	if !ok {
	48	+	t.Fatalf("netem type=%T, want *udpNetemRelay", netem)
	49	+	}
	50	+	relay.queueOverflowDrops.Store(3)
	51	+	if err := netem.Close(); err == nil \|\| err.Error() != "udp netem queue overflow drops: 3" {
	52	+	t.Fatalf("Close error=%v, want overflow rejection", err)
	53	+	}
	54	+	}
	55	+
	56	+	func TestUDPNetemExecutesRTTAndLossFixtures(t *testing.T) {
	57	+	fixtures := []struct {
	58	+	name string
	59	+	rtt time.Duration
	60	+	loss int
	61	+	sent int
	62	+	want int
	63	+	}{
	64	+	{"25ms RTT", 25 * time.Millisecond, 0, 1, 1},
	65	+	{"100ms RTT", 100 * time.Millisecond, 0, 1, 1},
	66	+	{"zero loss", 0, 0, 10, 10},
	67	+	{"one percent loss", 0, 1, 100, 99},
	68	+	}
	69	+	for _, tc := range fixtures {
	70	+	t.Run(tc.name, func(t *testing.T) {
	71	+	target, err := net.ListenPacket("udp", "127.0.0.1:0")
	72	+	if err != nil {
	73	+	t.Fatal(err)
	74	+	}
	75	+	closeUDPFixture(t, target)
	76	+	targetAddr, ok := target.LocalAddr().(*net.UDPAddr)
	77	+	if !ok {
	78	+	t.Fatalf("target address type=%T, want *net.UDPAddr", target.LocalAddr())
	79	+	}
	80	+	path := filepath.Join(t.TempDir(), "target")
	81	+	if err := os.WriteFile(path, []byte("VEV-UDP "+strconv.Itoa(targetAddr.Port)+" key\\n"), 0o600); err != nil {
	82	+	t.Fatal(err)
	83	+	}
	84	+	netem, err := newUDPNetem(udpNetemConfig{RTT: tc.rtt, LossPercent: tc.loss, TargetPath: path})
	85	+	if err != nil {
	86	+	t.Fatal(err)
	87	+	}
	88	+	closeUDPFixture(t, netem)
	89	+	client, err := net.ListenPacket("udp", "127.0.0.1:0")
	90	+	if err != nil {
	91	+	t.Fatal(err)
	92	+	}
	93	+	closeUDPFixture(t, client)
	94	+	started := time.Now()
	95	+	for i := 0; i < tc.sent; i++ {
	96	+	if _, err := client.WriteTo([]byte{byte(i)}, &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: netem.Port()}); err != nil {
	97	+	t.Fatal(err)
	98	+	}
	99	+	}
	100	+	if err := target.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
	101	+	t.Fatal(err)
	102	+	}
	103	+	got := 0
	104	+	buf := make([]byte, 32)
	105	+	var relayAddr net.Addr
	106	+	for got < tc.want {
	107	+	if _, relayAddr, err = target.ReadFrom(buf); err != nil {
	108	+	t.Fatalf("received %d packets, want %d: %v", got, tc.want, err)
	109	+	}
	110	+	got++
	111	+	}
	112	+	if tc.rtt > 0 {
	113	+	if elapsed := time.Since(started); elapsed < tc.rtt/2 {
	114	+	t.Fatalf("client-to-target netem delay=%s, want at least %s", elapsed, tc.rtt/2)
	115	+	}
	116	+	}
	117	+	if tc.loss == 0 {
	118	+	// The relay learns the client from the first carriage packet. Replying
	119	+	// through that same relay address verifies the target-to-client half
	120	+	// of the configured RTT as well.
	121	+	started = time.Now()
	122	+	if _, err := target.WriteTo([]byte("reply"), relayAddr); err != nil {
	123	+	t.Fatal(err)
	124	+	}
	125	+	if err := client.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
	126	+	t.Fatal(err)
	127	+	}
	128	+	if _, _, err := client.ReadFrom(buf); err != nil {
	129	+	t.Fatalf("relay reply: %v", err)
	130	+	}
	131	+	if tc.rtt > 0 {
	132	+	if elapsed := time.Since(started); elapsed < tc.rtt/2 {
	133	+	t.Fatalf("target-to-client netem delay=%s, want at least %s", elapsed, tc.rtt/2)
	134	+	}
	135	+	}
	136	+	}
	137	+	// Keep the deadline short: extra carriage would prove the exact 1%%
	138	+	// loss cycle was not enforced.
	139	+	if err := target.SetReadDeadline(time.Now().Add(20 * time.Millisecond)); err != nil {
	140	+	t.Fatal(err)
	141	+	}
	142	+	if _, _, err := target.ReadFrom(buf); tc.want != tc.sent && err == nil {
	143	+	t.Fatalf("received more than %d packets with %d%% loss", tc.want, tc.loss)
	144	+	}
	145	+	})
	146	+	}
	147	+	}

M docs/performance.md +63 -91

@@ -1,107 +1,79 @@
1	-	# Performance methodology (M5)	1	+	# Performance measurement
2	-		2	+
3	-	This baseline measures in-process daemon work only. It includes coordinator-enabled	3	+	## S5a transport observability
4	-	fixture scheduling metrics, but deliberately does not measure chunk	4	+
5	-	encoding, compression, GC tuning, or real network latency. Output metrics count frames and encoded output data produced	5	+	S5a is measurement-only. It records process-local runtime marks and harness-owned
6	-	by the daemon; snapshot writes and serialized bytes are local snapshot-store	6	+	end-to-end boundaries without changing rendering, pacing, transport policy, wire
7	-	proxies. These metrics approximate local rendering, IPC payload, and persistence	7	+	bytes, or `ProtocolVersion`. The canonical matrix and workload/topology/
8	-	work, not a Unix socket, SSH, or UDP transport. Real transport impairment and	8	+	transport coverage are in [`testdata/perf/manifest.json`](../testdata/perf/manifest.json):
9	-	latency benchmarks are deferred.	9	+	`1x4`, `4x1`, `4x4`, and `8x1`, all at `120x40` with 10,000 rows per pane;
10	-		10	+	idle, active/all/inactive output, interactive flood, copy search, resize sweep,
11	-	## Large-history daemon baseline	11	+	snapshot/output/resize, and attach/restore/tab switch; local, SSH stdio, UDP
12	-		12	+	baseline, UDP 25/100 ms, and UDP 0%/1% loss. It has 252 scenarios (4×9×7),
13	-	Use a single iteration only as a smoke check; it is not suitable for drawing	13	+	with inapplicable cases required to be explicit. The seven transport entries are
14	-	performance conclusions:	14	+	`local`, `ssh_stdio`, `udp_baseline`, `udp_loss_0pct`, `udp_25ms`, `udp_100ms`,
15	-		15	+	and `udp_loss_1pct`.
16	-	```sh	16	+
17	-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1x -benchmem	17	+	### Trace schema and clock domains
18	-	```	18	+
19	-		19	+	Every process JSONL record has exactly these fields:
20	-	For before/after comparison, collect repeated samples for the full matrix	20	+	`schema` (uint), `process_id` (string), `component` (string), `scenario`
21	-	(all workloads and topologies), pinning CPU/governor and recording the host.	21	+	(string), `run`, `sequence`, `request_id`, `epoch` (uint), `kind` (string),
22	-	The benchmark harness must be committed and present in both measured	22	+	`tick` (int64), `bytes`, `fragments`, `retransmits`, `pending` (uint),
23	-	revisions: collect the baseline after the harness is introduced but before the	23	+	`ack_rtt_nanos` (int64), and `valid` (bool). Schema 1 is represented by:
24	-	optimization, then collect the optimized revision with that same harness. The	24	+
25	-	fixed duration and count make the output suitable for `benchstat`:	25	+	```json
26	-		26	+	{"schema":1,"process_id":"p","component":"daemon","scenario":"s","run":1,"sequence":1,"request_id":1,"epoch":1,"kind":"diff_start","tick":0,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
27	-	```sh
28	-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1s -count=5 -benchmem > /tmp/vev-history-after.txt
29	-	benchstat /tmp/vev-history-before.txt /tmp/vev-history-after.txt
30		```	27		```
31			28
32	-	The matrix exercises live paint, snapshot capture, copy-mode entry, copy search,	29	+	`tick` is stamped only by the concrete JSONL observer using its process-local
33	-	and resize at `120x40` client geometry. Each pane is populated with 10,000	30	+	injected monotonic clock; producers supply no timestamp. The harness sets `VEV_PERF_TRACE`,
34	-	full-width deterministic history rows containing a stable `needle-<tab>-<row>`	31	+	`VEV_PERF_PROCESS_ID`, `VEV_PERF_SCENARIO`, and `VEV_PERF_RUN` for every
35	-	prefix and patterned text. The layouts are:	32	+	launched role, identifying its exclusive trace path and process/scenario/run
			33	+	mapping. Components in one OS
			34	+	process share that observer. The harness clock is a separate monotonic domain
			35	+	and is the sole clock for input-injection → terminal-bytes-successfully-flushed
			36	+	end-to-end samples. Cross-process ticks are never ordered or subtracted;
			37	+	records correlate only by scenario/run/sequence/request/epoch IDs.
36			38
37	-	- 1 tab x 1 pane (control)	39	+	The named process-local spans are capture, compose, diff, queue wait,
38	-	- 1 tab x 4 panes	40	+	ACK-blocked interval, emit, adapter send, and adapter receive. Their start/end
39	-	- 4 tabs x 1 pane	41	+	marks are paired within one process and summarized by component/adapter with
40	-	- 4 tabs x 4 panes	42	+	raw records, counts, p50/p95/p99, and max. Diagnostic fields (bytes, fragments,
41	-	- 8 tabs x 1 pane	43	+	retransmits, pending, ACK RTT) remain separate from duration calculations.
42			44
43	-	The fixture starts no PTY readers or transports. It primes the renderer shadow	45	+	### Baseline procedure and validity
44	-	before timing; live paint then alternates fixed writes through the coordinator.
45	-	The reported `outputframes/op` value counts output messages. `outputbytes/op`
46	-	counts their encoded terminal data and excludes the 24-byte output header;
47	-	`framepayloadbytes/op` counts the complete output payload including that header.
48	-	`snapshotwrites/op` and `snapshotbytes/op` are local snapshot-store writes and
49	-	serialized bytes. `coordinatorinvalidations/op`, `coordinatorwakes/op`, and
50	-	`coordinatorcoalesced/op` respectively count scheduling requests, delivered
51	-	wakes, and requests coalesced into those wakes; `coordinatorcoalescingratio`
52	-	is the average number of coalesced invalidations delivered per wake. They must
53	-	not be presented as network
54	-	throughput or latency.
55			46
56	-	Capture a pre-change result outside the repository, including Go and kernel	47	+	Use the public CLI harness (not internal packages):
57	-	context:
58			48
59		```sh	49		```sh
60	-	{	50	+	go build -o /tmp/vev-perf ./
61	-	date -Is	51	+	rm -rf testdata/perf/results
62	-	go version	52	+	go run ./cmd/vev-perf-harness --vev-bin /tmp/vev-perf \
63	-	uname -a	53	+	--manifest testdata/perf/manifest.json --out testdata/perf/results \
64	-	git rev-parse HEAD	54	+	--warmup 10s --duration 30s --repetitions 10
65	-	# Smoke only; do not use this result for comparisons.
66	-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1x -benchmem
67	-	# Repeated full matrix for every workload/topology used in conclusions.
68	-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1s -count=5 -benchmem
69	-	} > /tmp/vev-history-before.txt 2>&1
70		```	55		```
71			56
72	-	## Immutable document allocation gate	57	+	The minimums are 30 seconds measured and 10 independent repetitions; warmup is
			58	+	excluded. During the measured interval the harness injects at most one event per
			59	+	second. At least 10 complete event pairs are required: both injection and
			60	+	successful terminal flush must fall inside the measured interval; straddling or
			61	+	failed events are retained only as raw diagnostics and are not samples. Each run must contain `manifest.json`, `raw-harness.jsonl`, per-process
			62	+	JSONL, `runs.json`, and `summary.json`. Reject results with missing input/flush
			63	+	boundaries, unmatched IDs, bad correlation, nonmonotonic same-process sequence,
			64	+	negative same-domain spans, fewer than 10 complete in-interval event pairs,
			65	+	invalid denominators,
			66	+	any cross-process tick arithmetic, or unmet duration/repetition minima. Raw
			67	+	JSONL and run manifests are the evidence files; no result authorizes a policy
			68	+	or optimization change.
73			69
74	-	The copy-mode document retains an immutable view of existing scrollback rows and	70	+	## Existing in-process smoke checks
75	-	clones only the visible frame. Visual-search models share that document; cloning
76	-	a model copies only its mutable input and match state. Check those two paths at
77	-	10,000 history rows with:
78			71
79	-	```sh	72	+	These commands measure local daemon work only and are not end-to-end transport
80	-	go test ./internal/usecase/copy ./internal/usecase/visualsearch -run '^$' -bench '^(BenchmarkNewSnapshot10KRows\|BenchmarkVisualSearchClone10KRows)$' -benchtime=1s -count=5 -benchmem	73	+	results:
81	-	```
82	-
83	-	Treat `allocs/op` and `B/op` as allocation gates: compare the complete repeated
84	-	output with the accepted baseline before merging, and investigate any increase.
85	-	The automated scaling gates also compare equal row/match counts at narrow and
86	-	wide row widths; `B/op` may grow by at most 2x, a conservative relative
87	-	allowance that catches full cell-row copies without depending on an absolute
88	-	Go-version-specific budget. This scope does not yet include an
89	-	incremental/full-text search index: queries still scan the immutable document
90	-	and produce a fresh match list. Chunk encoding, compression, GC tuning, and real Unix-socket, SSH, or UDP
91	-	impairment/latency remain out of scope.
92	-
93	-	## Lower-level smoke baseline
94	-
95	-	Use this command to check renderer, VT, IPC, and daemon microbenchmarks without
96	-	claiming an end-to-end transport result:
97			74
98		```sh	75		```sh
			76	+	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1x -benchmem
99		go test ./pkg/renderer ./pkg/vt ./internal/adapters/ipc -run '^$' -bench=. -benchmem	77		go test ./pkg/renderer ./pkg/vt ./internal/adapters/ipc -run '^$' -bench=. -benchmem
100		go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkComposeFloatingFrameCached$' -benchmem	78		go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkComposeFloatingFrameCached$' -benchmem
101		```	79		```
102	-
103	-	Enable daemon pprof explicitly when collecting a daemon profile:
104	-
105	-	```sh
106	-	VEV_PPROF_ADDR=127.0.0.1:6060 vev --daemon
107	-	```

M internal/adapters/dgram/ssh_dialer.go +2 -1

 	BootstrapTimeout time.Duration
 	ProbeTimeout     time.Duration
 	Log              *slog.Logger
+	RuntimeObserver  ports.RuntimeObserver
 }
 func NewRemoteDialer(target, session string) RemoteDialer {
 		RebindPacketConn: func(net.PacketConn) (net.PacketConn, error) {
 			return listenUDPPacket(ctx)
 		},
-	})
+	}, WithRuntimeObserver(d.RuntimeObserver))
 	if err != nil {
 		_ = pc.Close()
 		return nil, udpUnavailable("create UDP transport", err, &stderr)

M internal/adapters/dgram/transport.go +96 -7


@@ -45,6 +45,14 @@ var (
45		errControlFull = errors.New("dgram: control queue full")	45		errControlFull = errors.New("dgram: control queue full")
46		)	46		)
47			47
			48	+	type RuntimeOption func(*Transport)
			49	+
			50	+	// WithRuntimeObserver enables process-local marks without changing dgram's
			51	+	// behavior clock or carriage policy.
			52	+	func WithRuntimeObserver(observer ports.RuntimeObserver) RuntimeOption {
			53	+	return func(t *Transport) { t.runtimeObserver = observer }
			54	+	}
			55	+
48		type Options struct {	56		type Options struct {
49		MTU int	57		MTU int
50		ResendAfter time.Duration	58		ResendAfter time.Duration

@@ -168,6 +176,11 @@ type Transport struct {
168		// mutation and before ordered event publication.	176		// mutation and before ordered event publication.
169		afterLinkStateCommit func(ports.LinkState)	177		afterLinkStateCommit func(ports.LinkState)
170		observe DiagnosticObserver	178		observe DiagnosticObserver
			179	+	runtimeObserver ports.RuntimeObserver
			180	+	operationMu sync.Mutex
			181	+	operationCount int
			182	+	closing bool
			183	+	operationsDone chan struct{}
171		diagnosticCh chan Diagnostic	184		diagnosticCh chan Diagnostic
172		heartbeat time.Duration	185		heartbeat time.Duration
173		resendAfter time.Duration	186		resendAfter time.Duration

@@ -311,11 +324,11 @@ func (s *writeDeadlineState) expire(now time.Time) {
311		s.mu.Unlock()	324		s.mu.Unlock()
312		}	325		}
313			326
314	-	func NewTransport(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32) (*Transport, error) {	327	+	func NewTransport(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32, runtimeOpts ...RuntimeOption) (*Transport, error) {
315	-	return NewTransportWithOptions(pc, peer, key, sendDir, recvDir, Options{})	328	+	return NewTransportWithOptions(pc, peer, key, sendDir, recvDir, Options{}, runtimeOpts...)
316		}	329		}
317			330
318	-	func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32, opts Options) (*Transport, error) {	331	+	func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32, opts Options, runtimeOpts ...RuntimeOption) (*Transport, error) {
319		c, err := pdgram.NewCodec(key)	332		c, err := pdgram.NewCodec(key)
320		if err != nil {	333		if err != nil {
321		return nil, err	334		return nil, err

@@ -360,6 +373,11 @@ func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendD
360		nextRecvSeq: 1,	373		nextRecvSeq: 1,
361		recvBuf: make(map[uint64]ports.Frame),	374		recvBuf: make(map[uint64]ports.Frame),
362		}	375		}
			376	+	for _, opt := range runtimeOpts {
			377	+	if opt != nil {
			378	+	opt(t)
			379	+	}
			380	+	}
363		t.sendWake = make(chan struct{})	381		t.sendWake = make(chan struct{})
364		t.control = make(chan controlRecord, controlQueueSize)	382		t.control = make(chan controlRecord, controlQueueSize)
365		t.ackWake = make(chan struct{}, 1)	383		t.ackWake = make(chan struct{}, 1)

@@ -387,9 +405,19 @@ func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendD
387		return t, nil	405		return t, nil
388		}	406		}
389			407
390	-	func (t *Transport) Send(f ports.Frame) error { return t.send(f, false) }	408	+	func (t *Transport) Send(f ports.Frame) error {
			409	+	end := t.beginRuntimeOperation(ports.RuntimeAdapterSendStart, uint64(len(f.Payload)))
			410	+	err := t.send(f, false)
			411	+	end(err == nil)
			412	+	return err
			413	+	}
391			414
392	-	func (t *Transport) SendAsync(f ports.Frame) error { return t.send(f, true) }	415	+	func (t *Transport) SendAsync(f ports.Frame) error {
			416	+	end := t.beginRuntimeOperation(ports.RuntimeAdapterSendStart, uint64(len(f.Payload)))
			417	+	err := t.send(f, true)
			418	+	end(err == nil)
			419	+	return err
			420	+	}
393			421
394		// SendSynchronous owns queue reservation, fragment pacing, per-write deadlines,	422		// SendSynchronous owns queue reservation, fragment pacing, per-write deadlines,
395		// and close cancellation. Callers must not layer a shorter preflight timer over it.	423		// and close cancellation. Callers must not layer a shorter preflight timer over it.

@@ -568,25 +596,35 @@ func (t *Transport) removePendingLocked(seq uint64) bool {
568		}	596		}
569			597
570		func (t *Transport) Recv() (ports.Frame, error) {	598		func (t *Transport) Recv() (ports.Frame, error) {
			599	+	end := t.beginRuntimeOperation(ports.RuntimeAdapterReceiveStart, 0)
571		select {	600		select {
572		case f := <-t.in:	601		case f := <-t.in:
			602	+	end(true)
573		return f, nil	603		return f, nil
574		case <-t.done:	604		case <-t.done:
575		t.mu.Lock()	605		t.mu.Lock()
576		err := t.closeErr	606		err := t.closeErr
577		t.mu.Unlock()	607		t.mu.Unlock()
578		if err != nil {	608		if err != nil {
			609	+	end(false)
579		return ports.Frame{}, err	610		return ports.Frame{}, err
580		}	611		}
581	-	return ports.Frame{}, errors.New("dgram: closed")	612	+	err = errors.New("dgram: closed")
			613	+	end(false)
			614	+	return ports.Frame{}, err
582		}	615		}
583		}	616		}
584		func (t *Transport) Close() error {	617		func (t *Transport) Close() error {
			618	+	done := t.beginShutdown()
585		t.closeWithError(errors.New("dgram: closed"))	619		t.closeWithError(errors.New("dgram: closed"))
586		t.mu.Lock()	620		t.mu.Lock()
587		pc := t.pc	621		pc := t.pc
588		t.mu.Unlock()	622		t.mu.Unlock()
589	-	return pc.Close()	623	+	err := pc.Close()
			624	+	if done != nil {
			625	+	<-done
			626	+	}
			627	+	return err
590		}	628		}
591			629
592		func (t *Transport) Peer() net.Addr { t.mu.Lock(); defer t.mu.Unlock(); return t.peer }	630		func (t *Transport) Peer() net.Addr { t.mu.Lock(); defer t.mu.Unlock(); return t.peer }

@@ -1585,4 +1623,55 @@ func decodeData(b []byte) (uint64, bool, ports.Frame, bool) {
1585		}	1623		}
1586			1624
1587		// DatagramTransport marks Transport as a UDP-style datagram transport.	1625		// DatagramTransport marks Transport as a UDP-style datagram transport.
			1626	+	func (t *Transport) beginRuntimeOperation(start ports.RuntimeMarkKind, bytes uint64) func(bool) {
			1627	+	if t.runtimeObserver == nil {
			1628	+	return func(bool) {}
			1629	+	}
			1630	+	if !t.beginObservedOperation() {
			1631	+	return func(bool) {}
			1632	+	}
			1633	+	correlation := ports.NewRuntimeCorrelation()
			1634	+	t.runtimeObserver.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("dgram", correlation, start, bytes, true))
			1635	+	end := ports.RuntimeAdapterSendEnd
			1636	+	if start == ports.RuntimeAdapterReceiveStart {
			1637	+	end = ports.RuntimeAdapterReceiveEnd
			1638	+	}
			1639	+	return func(valid bool) {
			1640	+	defer t.finishObservedOperation()
			1641	+	t.runtimeObserver.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("dgram", correlation, end, bytes, valid))
			1642	+	}
			1643	+	}
			1644	+
			1645	+	func (t *Transport) beginObservedOperation() bool {
			1646	+	t.operationMu.Lock()
			1647	+	defer t.operationMu.Unlock()
			1648	+	if t.closing {
			1649	+	return false
			1650	+	}
			1651	+	if t.operationCount == 0 {
			1652	+	t.operationsDone = make(chan struct{})
			1653	+	}
			1654	+	t.operationCount++
			1655	+	return true
			1656	+	}
			1657	+
			1658	+	func (t *Transport) finishObservedOperation() {
			1659	+	t.operationMu.Lock()
			1660	+	defer t.operationMu.Unlock()
			1661	+	t.operationCount--
			1662	+	if t.operationCount == 0 {
			1663	+	close(t.operationsDone)
			1664	+	}
			1665	+	}
			1666	+
			1667	+	func (t *Transport) beginShutdown() <-chan struct{} {
			1668	+	t.operationMu.Lock()
			1669	+	defer t.operationMu.Unlock()
			1670	+	t.closing = true
			1671	+	if t.operationCount == 0 {
			1672	+	return nil
			1673	+	}
			1674	+	return t.operationsDone
			1675	+	}
			1676	+
1588		func (*Transport) DatagramTransport() {}	1677		func (*Transport) DatagramTransport() {}

A internal/adapters/dgram/transport_observability_test.go +202 -0

@@ -0,0 +1,202 @@
	1	+	package dgram
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"errors"
	6	+	"net"
	7	+	"os"
	8	+	"reflect"
	9	+	"strings"
	10	+	"sync"
	11	+	"testing"
	12	+	"time"
	13	+
	14	+	"github.com/bnema/vev/internal/adapters/observability"
	15	+	"github.com/bnema/vev/internal/ports"
	16	+	)
	17	+
	18	+	func TestTransportObservabilityDgramFailedCloseEndsReceiveAndSend(t *testing.T) {
	19	+	behaviorClock := newManualClock(time.Unix(0, 417))
	20	+	link := newSimulatedLink(behaviorClock, packetPolicy{})
	21	+	aPC, _ := newPairWithCapacity(link, 64)
	22	+	closeErr := errors.New("packet connection close failed")
	23	+	tracePath := t.TempDir() + "/dgram-close.jsonl"
	24	+	observer, closer, err := observability.NewJSONL(tracePath, dgramObserverClock{now: time.Unix(0, 521)}, "dgram-process")
	25	+	if err != nil {
	26	+	t.Fatalf("NewJSONL() error = %v", err)
	27	+	}
	28	+	receiveStarted := make(chan struct{})
	29	+	signalingObserver := &dgramStartObserver{RuntimeObserver: observer, receiveStarted: receiveStarted}
	30	+	transport, err := NewTransportWithOptions(&dgramCloseFailPC{fakePC: aPC, err: closeErr}, testAddr("b"), key(), 1, 2, Options{Clock: behaviorClock, ResendAfter: time.Hour, Heartbeat: time.Hour}, WithRuntimeObserver(signalingObserver))
	31	+	if err != nil {
	32	+	t.Fatalf("NewTransportWithOptions() error = %v", err)
	33	+	}
	34	+	if err := transport.Send(ports.Frame{Type: ports.MsgOutput, Payload: []byte("write failure")}); err == nil {
	35	+	t.Fatal("Send() error = nil on packet write failure")
	36	+	}
	37	+	recvDone := make(chan error, 1)
	38	+	go func() { _, err := transport.Recv(); recvDone <- err }()
	39	+	<-receiveStarted
	40	+	if err := transport.Close(); !errors.Is(err, closeErr) {
	41	+	t.Fatalf("Close() error = %v, want %v", err, closeErr)
	42	+	}
	43	+	if err := <-recvDone; err == nil {
	44	+	t.Fatal("Recv() error = nil after failed Close()")
	45	+	}
	46	+	if err := closer.Close(); err != nil {
	47	+	t.Fatalf("trace Close() error = %v", err)
	48	+	}
	49	+	assertDgramFailedSpanPairs(t, tracePath)
	50	+	}
	51	+
	52	+	type dgramStartObserver struct {
	53	+	ports.RuntimeObserver
	54	+	receiveStarted chan struct{}
	55	+	once sync.Once
	56	+	}
	57	+
	58	+	func (o *dgramStartObserver) ObserveRuntime(mark ports.RuntimeMark) {
	59	+	if mark.Kind == ports.RuntimeAdapterReceiveStart {
	60	+	o.once.Do(func() { close(o.receiveStarted) })
	61	+	}
	62	+	o.RuntimeObserver.ObserveRuntime(mark)
	63	+	}
	64	+
	65	+	type dgramCloseFailPC struct {
	66	+	*fakePC
	67	+	err error
	68	+	}
	69	+
	70	+	func (p *dgramCloseFailPC) WriteTo([]byte, net.Addr) (int, error) {
	71	+	return 0, errors.New("packet write failed")
	72	+	}
	73	+
	74	+	func (p *dgramCloseFailPC) Close() error {
	75	+	_ = p.fakePC.Close()
	76	+	return p.err
	77	+	}
	78	+
	79	+	func assertDgramFailedSpanPairs(t *testing.T, path string) {
	80	+	t.Helper()
	81	+	data, err := os.ReadFile(path)
	82	+	if err != nil {
	83	+	t.Fatalf("ReadFile(trace): %v", err)
	84	+	}
	85	+	type mark struct {
	86	+	ProcessID string `json:"process_id"`
	87	+	Scenario string `json:"scenario"`
	88	+	Run uint64 `json:"run"`
	89	+	Sequence uint64 `json:"sequence"`
	90	+	RequestID uint64 `json:"request_id"`
	91	+	Epoch uint64 `json:"epoch"`
	92	+	Kind string `json:"kind"`
	93	+	Valid bool `json:"valid"`
	94	+	}
	95	+	type spanKey struct {
	96	+	processID, scenario string
	97	+	run, sequence, requestID, epoch uint64
	98	+	kind string
	99	+	}
	100	+	starts := make(map[spanKey]mark)
	101	+	pairs := map[string]int{}
	102	+	for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
	103	+	var m mark
	104	+	if err := json.Unmarshal([]byte(line), &m); err != nil {
	105	+	t.Fatalf("Unmarshal(trace): %v", err)
	106	+	}
	107	+	key := spanKey{m.ProcessID, m.Scenario, m.Run, m.Sequence, m.RequestID, m.Epoch, m.Kind}
	108	+	switch m.Kind {
	109	+	case "adapter_receive_start", "adapter_send_start":
	110	+	if _, exists := starts[key]; exists {
	111	+	t.Fatalf("duplicate adapter start: %#v", m)
	112	+	}
	113	+	starts[key] = m
	114	+	case "adapter_receive_end", "adapter_send_end":
	115	+	key.kind = strings.TrimSuffix(m.Kind, "_end") + "_start"
	116	+	start, ok := starts[key]
	117	+	if !ok \|\| m.Valid {
	118	+	t.Fatalf("unmatched or valid failed end: start=%#v end=%#v", start, m)
	119	+	}
	120	+	pairs[strings.TrimSuffix(m.Kind, "_end")]++
	121	+	delete(starts, key)
	122	+	}
	123	+	}
	124	+	if len(starts) != 0 {
	125	+	t.Fatalf("unmatched adapter starts: %#v", starts)
	126	+	}
	127	+	if pairs["adapter_send"] != 1 \|\| pairs["adapter_receive"] != 1 \|\| len(pairs) != 2 {
	128	+	t.Fatalf("failed adapter span pairs=%v, want exactly one send and one receive", pairs)
	129	+	}
	130	+	}
	131	+
	132	+	func TestTransportObservabilityDgramKeepsBehaviorClockSeparate(t *testing.T) {
	133	+	behaviorClock := newManualClock(time.Unix(0, 401))
	134	+	link := newSimulatedLink(behaviorClock, packetPolicy{})
	135	+	aPC, bPC := newPairWithCapacity(link, 64)
	136	+	tracePath := t.TempDir() + "/dgram.jsonl"
	137	+	observer, closer, err := observability.NewJSONL(tracePath, dgramObserverClock{now: time.Unix(0, 509)}, "dgram-process")
	138	+	if err != nil {
	139	+	t.Fatalf("NewJSONL() error = %v", err)
	140	+	}
	141	+
	142	+	a, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{Clock: behaviorClock, ResendAfter: time.Hour, Heartbeat: time.Hour}, WithRuntimeObserver(observer))
	143	+	if err != nil {
	144	+	t.Fatalf("NewTransportWithOptions(a) error = %v", err)
	145	+	}
	146	+	defer func() { _ = a.Close() }()
	147	+	b, err := NewTransportWithOptions(bPC, testAddr("a"), key(), 2, 1, Options{Clock: behaviorClock, ResendAfter: time.Hour, Heartbeat: time.Hour}, WithRuntimeObserver(observer))
	148	+	if err != nil {
	149	+	t.Fatalf("NewTransportWithOptions(b) error = %v", err)
	150	+	}
	151	+	defer func() { _ = b.Close() }()
	152	+
	153	+	want := ports.Frame{Type: ports.MsgOutput, Payload: []byte("datagram carriage remains exact")}
	154	+	if err := a.Send(want); err != nil {
	155	+	t.Fatalf("Send() error = %v", err)
	156	+	}
	157	+	gotCh := make(chan ports.Frame, 1)
	158	+	errCh := make(chan error, 1)
	159	+	go func() {
	160	+	frame, recvErr := b.Recv()
	161	+	if recvErr != nil {
	162	+	errCh <- recvErr
	163	+	return
	164	+	}
	165	+	gotCh <- frame
	166	+	}()
	167	+	select {
	168	+	case err := <-errCh:
	169	+	t.Fatalf("Recv() error = %v", err)
	170	+	case got := <-gotCh:
	171	+	if !reflect.DeepEqual(got, want) {
	172	+	t.Fatalf("carried frame = %#v, want %#v", got, want)
	173	+	}
	174	+	case <-time.After(time.Second):
	175	+	t.Fatal("Recv() did not receive deterministic simulated datagram")
	176	+	}
	177	+	if err := closer.Close(); err != nil {
	178	+	t.Fatalf("trace Close() error = %v", err)
	179	+	}
	180	+	trace, err := os.ReadFile(tracePath)
	181	+	if err != nil {
	182	+	t.Fatalf("ReadFile(trace): %v", err)
	183	+	}
	184	+	for _, kind := range []string{"adapter_send_start", "adapter_send_end", "adapter_receive_start", "adapter_receive_end"} {
	185	+	if !strings.Contains(string(trace), `"kind":"`+kind+`"`) {
	186	+	t.Errorf("trace missing %s", kind)
	187	+	}
	188	+	}
	189	+	}
	190	+
	191	+	type dgramObserverClock struct{ now time.Time }
	192	+
	193	+	func (c dgramObserverClock) Now() time.Time { return c.now }
	194	+	func (c dgramObserverClock) NewTimer(d time.Duration) ports.Timer {
	195	+	return dgramObserverTimer{timer: time.NewTimer(d)}
	196	+	}
	197	+
	198	+	type dgramObserverTimer struct{ timer *time.Timer }
	199	+
	200	+	func (t dgramObserverTimer) C() <-chan time.Time { return t.timer.C }
	201	+	func (t dgramObserverTimer) Reset(d time.Duration) bool { return t.timer.Reset(d) }
	202	+	func (t dgramObserverTimer) Stop() bool { return t.timer.Stop() }

M internal/adapters/ipc/dial.go +7 -5

 package ipc
 import (
+	"context"
 	"fmt"
 	"net"
 	"path/filepath"
 	"github.com/bnema/vev/internal/ports"
 )
-// Dial connects to the daemon's socket in dir (dir/daemon.sock) and returns
+// DialContext connects to the daemon's socket in dir (dir/daemon.sock) and
-// a Transport speaking vev's framed protocol. A dial failure (no daemon
+// returns a Transport speaking vev's framed protocol. A dial failure (no daemon
 // listening, or a stale/absent socket) is reported as a wrapped error so
 // callers can decide whether to spawn a daemon and retry.
-func Dial(dir string) (ports.Transport, error) {
+func DialContext(ctx context.Context, dir string, opts ...Option) (ports.Transport, error) {
 	sockPath := filepath.Join(dir, socketFileName)
-	conn, err := net.Dial("unix", sockPath)
+	var dialer net.Dialer
+	conn, err := dialer.DialContext(ctx, "unix", sockPath)
 	if err != nil {
 		return nil, fmt.Errorf("ipc: dial %s: %w", sockPath, err)
 	}
-	return NewTransport(conn), nil
+	return NewTransport(conn, opts...), nil
 }

A internal/adapters/ipc/dial_test.go +79 -0

@@ -0,0 +1,79 @@
	1	+	package ipc
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"path/filepath"
	7	+	"testing"
	8	+	"time"
	9	+
	10	+	"github.com/bnema/vev/internal/ports"
	11	+	)
	12	+
	13	+	type dialTestObserver struct{}
	14	+
	15	+	func (dialTestObserver) ObserveRuntime(ports.RuntimeMark) {}
	16	+
	17	+	func TestDialContextCanceledOrExpired(t *testing.T) {
	18	+	tests := []struct {
	19	+	name string
	20	+	ctx func() (context.Context, context.CancelFunc)
	21	+	want error
	22	+	}{
	23	+	{
	24	+	name: "canceled",
	25	+	ctx: func() (context.Context, context.CancelFunc) {
	26	+	ctx, cancel := context.WithCancel(context.Background())
	27	+	cancel()
	28	+	return ctx, func() {}
	29	+	},
	30	+	want: context.Canceled,
	31	+	},
	32	+	{
	33	+	name: "deadline exceeded",
	34	+	ctx: func() (context.Context, context.CancelFunc) {
	35	+	return context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
	36	+	},
	37	+	want: context.DeadlineExceeded,
	38	+	},
	39	+	}
	40	+
	41	+	for _, tt := range tests {
	42	+	t.Run(tt.name, func(t *testing.T) {
	43	+	ctx, cancel := tt.ctx()
	44	+	defer cancel()
	45	+
	46	+	transport, err := DialContext(ctx, t.TempDir())
	47	+	if transport != nil {
	48	+	t.Fatal("DialContext returned a transport for a completed context")
	49	+	}
	50	+	if !errors.Is(err, tt.want) {
	51	+	t.Fatalf("DialContext error = %v, want wrapping %v", err, tt.want)
	52	+	}
	53	+	})
	54	+	}
	55	+	}
	56	+
	57	+	func TestDialContextPreservesOptions(t *testing.T) {
	58	+	dir := filepath.Join(t.TempDir(), "vev")
	59	+	listener, err := Listen(dir)
	60	+	if err != nil {
	61	+	t.Fatalf("Listen: %v", err)
	62	+	}
	63	+	defer func() { _ = listener.Close() }()
	64	+
	65	+	observer := dialTestObserver{}
	66	+	transport, err := DialContext(context.Background(), dir, WithRuntimeObserver(observer))
	67	+	if err != nil {
	68	+	t.Fatalf("DialContext: %v", err)
	69	+	}
	70	+	defer func() { _ = transport.Close() }()
	71	+
	72	+	got, ok := transport.(*unixTransport)
	73	+	if !ok {
	74	+	t.Fatalf("DialContext transport = %T, want *unixTransport", transport)
	75	+	}
	76	+	if got.observer != observer {
	77	+	t.Fatal("DialContext did not apply runtime observer option")
	78	+	}
	79	+	}

M internal/adapters/ipc/listener.go +4 -3

 type unixListener struct {
 	ln   *net.UnixListener
 	addr string
+	opts []Option
 }
 // Listen creates the socket directory (0700) if needed and starts
 // the previous owner died without cleaning up, so the stale socket file is
 // unlinked and bind is retried once. A successful dial means a live daemon
 // owns the socket, and Listen returns ErrDaemonRunning.
-func Listen(dir string) (ports.Listener, error) {
+func Listen(dir string, opts ...Option) (ports.Listener, error) {
 	if err := safedir.EnsurePrivate(dir); err != nil {
 		return nil, fmt.Errorf("ipc: securing socket directory: %w", err)
 	}
 		return nil, fmt.Errorf("ipc: chmod %s: %w", sockPath, err)
 	}
-	return &unixListener{ln: ln, addr: sockPath}, nil
+	return &unixListener{ln: ln, addr: sockPath, opts: opts}, nil
 }
 // bindUnix binds and listens on sockPath.
 	if err != nil {
 		return nil, err
 	}
-	return NewTransport(conn), nil
+	return NewTransport(conn, l.opts...), nil
 }
 // Close stops accepting connections. The underlying net.UnixListener

M internal/adapters/ipc/transport.go +91 -5


@@ -5,6 +5,7 @@ import (
5		"errors"	5		"errors"
6		"io"	6		"io"
7		"net"	7		"net"
			8	+	"sync"
8			9
9		"github.com/bnema/vev/internal/ports"	10		"github.com/bnema/vev/internal/ports"
10		)	11		)

@@ -35,8 +36,21 @@ var ErrFrameTooLarge = errors.New("ipc: frame exceeds maximum length")
35		// goroutine and a single writer goroutine); it is not safe for concurrent	36		// goroutine and a single writer goroutine); it is not safe for concurrent
36		// calls to Send from multiple goroutines, nor Recv from multiple	37		// calls to Send from multiple goroutines, nor Recv from multiple
37		// goroutines.	38		// goroutines.
			39	+	type Option func(*unixTransport)
			40	+
			41	+	// WithRuntimeObserver enables process-local transport marks. It accepts no
			42	+	// clock: timestamp ownership belongs to the configured observer.
			43	+	func WithRuntimeObserver(observer ports.RuntimeObserver) Option {
			44	+	return func(t *unixTransport) { t.observer = observer }
			45	+	}
			46	+
38		type unixTransport struct {	47		type unixTransport struct {
39	-	conn net.Conn	48	+	conn net.Conn
			49	+	observer ports.RuntimeObserver
			50	+	operationMu sync.Mutex
			51	+	operationCount int
			52	+	closing bool
			53	+	operationsDone chan struct{}
40			54
41		// readBuf is reused across Recv calls (grow-once strategy): it grows	55		// readBuf is reused across Recv calls (grow-once strategy): it grows
42		// to fit the largest frame seen so far and is never shrunk. The Frame	56		// to fit the largest frame seen so far and is never shrunk. The Frame

@@ -47,16 +61,24 @@ type unixTransport struct {
47			61
48		// NewTransport wraps conn as a ports.Transport speaking vev's framed	62		// NewTransport wraps conn as a ports.Transport speaking vev's framed
49		// binary protocol.	63		// binary protocol.
50	-	func NewTransport(conn net.Conn) ports.Transport {	64	+	func NewTransport(conn net.Conn, opts ...Option) ports.Transport {
51	-	return &unixTransport{conn: conn}	65	+	t := &unixTransport{conn: conn}
			66	+	for _, opt := range opts {
			67	+	if opt != nil {
			68	+	opt(t)
			69	+	}
			70	+	}
			71	+	return t
52		}	72		}
53			73
54		// Send writes f as a single frame: a 4-byte big-endian length (covering	74		// Send writes f as a single frame: a 4-byte big-endian length (covering
55		// the type byte and payload), the type byte, then the payload — assembled	75		// the type byte and payload), the type byte, then the payload — assembled
56		// into one buffer and written with a single Write call.	76		// into one buffer and written with a single Write call.
57		func (t *unixTransport) Send(f ports.Frame) error {	77		func (t *unixTransport) Send(f ports.Frame) error {
			78	+	end := t.beginOperation(ports.RuntimeAdapterSendStart, uint64(len(f.Payload)))
58		n := 1 + len(f.Payload) // type + payload	79		n := 1 + len(f.Payload) // type + payload
59		if n > maxFrameLen {	80		if n > maxFrameLen {
			81	+	end(false)
60		return ErrFrameTooLarge	82		return ErrFrameTooLarge
61		}	83		}
62			84

@@ -66,6 +88,7 @@ func (t *unixTransport) Send(f ports.Frame) error {
66		copy(buf[frameHeaderLen+1:], f.Payload)	88		copy(buf[frameHeaderLen+1:], f.Payload)
67			89
68		_, err := t.conn.Write(buf)	90		_, err := t.conn.Write(buf)
			91	+	end(err == nil)
69		return err	92		return err
70		}	93		}
71			94

@@ -73,16 +96,20 @@ func (t *unixTransport) Send(f ports.Frame) error {
73		// (type byte + payload). It blocks until a full frame arrives, the	96		// (type byte + payload). It blocks until a full frame arrives, the
74		// connection is closed (io.EOF), or an error occurs.	97		// connection is closed (io.EOF), or an error occurs.
75		func (t *unixTransport) Recv() (ports.Frame, error) {	98		func (t *unixTransport) Recv() (ports.Frame, error) {
			99	+	end := t.beginOperation(ports.RuntimeAdapterReceiveStart, 0)
76		var hdr [frameHeaderLen]byte	100		var hdr [frameHeaderLen]byte
77		if _, err := io.ReadFull(t.conn, hdr[:]); err != nil {	101		if _, err := io.ReadFull(t.conn, hdr[:]); err != nil {
			102	+	end(false)
78		return ports.Frame{}, err	103		return ports.Frame{}, err
79		}	104		}
80			105
81		n := binary.BigEndian.Uint32(hdr[:])	106		n := binary.BigEndian.Uint32(hdr[:])
82		if n == 0 {	107		if n == 0 {
			108	+	end(false)
83		return ports.Frame{}, ErrZeroLengthFrame	109		return ports.Frame{}, ErrZeroLengthFrame
84		}	110		}
85		if n > maxFrameLen {	111		if n > maxFrameLen {
			112	+	end(false)
86		return ports.Frame{}, ErrFrameTooLarge	113		return ports.Frame{}, ErrFrameTooLarge
87		}	114		}
88			115

@@ -92,6 +119,7 @@ func (t *unixTransport) Recv() (ports.Frame, error) {
92		t.readBuf = t.readBuf[:n]	119		t.readBuf = t.readBuf[:n]
93		}	120		}
94		if _, err := io.ReadFull(t.conn, t.readBuf); err != nil {	121		if _, err := io.ReadFull(t.conn, t.readBuf); err != nil {
			122	+	end(false)
95		return ports.Frame{}, err	123		return ports.Frame{}, err
96		}	124		}
97			125

@@ -100,10 +128,68 @@ func (t *unixTransport) Recv() (ports.Frame, error) {
100		payload = make([]byte, n-1)	128		payload = make([]byte, n-1)
101		copy(payload, t.readBuf[1:])	129		copy(payload, t.readBuf[1:])
102		}	130		}
			131	+	end(true)
103		return ports.Frame{Type: ports.MsgType(t.readBuf[0]), Payload: payload}, nil	132		return ports.Frame{Type: ports.MsgType(t.readBuf[0]), Payload: payload}, nil
104		}	133		}
105			134
106	-	// Close closes the underlying connection.	135	+	func (t *unixTransport) beginOperation(start ports.RuntimeMarkKind, bytes uint64) func(bool) {
			136	+	if t.observer == nil {
			137	+	return func(bool) {}
			138	+	}
			139	+	if !t.beginObservedOperation() {
			140	+	return func(bool) {}
			141	+	}
			142	+	correlation := ports.NewRuntimeCorrelation()
			143	+	t.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("ipc", correlation, start, bytes, true))
			144	+	end := ports.RuntimeAdapterSendEnd
			145	+	if start == ports.RuntimeAdapterReceiveStart {
			146	+	end = ports.RuntimeAdapterReceiveEnd
			147	+	}
			148	+	return func(valid bool) {
			149	+	defer t.finishObservedOperation()
			150	+	t.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("ipc", correlation, end, bytes, valid))
			151	+	}
			152	+	}
			153	+
			154	+	func (t *unixTransport) beginObservedOperation() bool {
			155	+	t.operationMu.Lock()
			156	+	defer t.operationMu.Unlock()
			157	+	if t.closing {
			158	+	return false
			159	+	}
			160	+	if t.operationCount == 0 {
			161	+	t.operationsDone = make(chan struct{})
			162	+	}
			163	+	t.operationCount++
			164	+	return true
			165	+	}
			166	+
			167	+	func (t *unixTransport) finishObservedOperation() {
			168	+	t.operationMu.Lock()
			169	+	defer t.operationMu.Unlock()
			170	+	t.operationCount--
			171	+	if t.operationCount == 0 {
			172	+	close(t.operationsDone)
			173	+	}
			174	+	}
			175	+
			176	+	func (t *unixTransport) beginShutdown() <-chan struct{} {
			177	+	t.operationMu.Lock()
			178	+	defer t.operationMu.Unlock()
			179	+	t.closing = true
			180	+	if t.operationCount == 0 {
			181	+	return nil
			182	+	}
			183	+	return t.operationsDone
			184	+	}
			185	+
			186	+	// Close closes the underlying connection and waits for any interrupted
			187	+	// carriage operation to write its matching failed end mark.
107		func (t *unixTransport) Close() error {	188		func (t *unixTransport) Close() error {
108	-	return t.conn.Close()	189	+	done := t.beginShutdown()
			190	+	err := t.conn.Close()
			191	+	if done != nil {
			192	+	<-done
			193	+	}
			194	+	return err
109		}	195		}

A internal/adapters/ipc/transport_observability_test.go +191 -0

@@ -0,0 +1,191 @@
	1	+	package ipc
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"net"
	6	+	"os"
	7	+	"reflect"
	8	+	"strings"
	9	+	"sync"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"github.com/bnema/vev/internal/adapters/observability"
	14	+	"github.com/bnema/vev/internal/ports"
	15	+	)
	16	+
	17	+	func TestTransportObservabilityIPCEOFAndCloseEndFailedSpans(t *testing.T) {
	18	+	trace := t.TempDir() + "/ipc-shutdown.jsonl"
	19	+	observer, closer, err := observability.NewJSONL(trace, ipcRuntimeClock{now: time.Unix(0, 223)}, "ipc-process")
	20	+	if err != nil {
	21	+	t.Fatalf("NewJSONL() error = %v", err)
	22	+	}
	23	+
	24	+	// EOF from the peer and a locally closed blocked receive are distinct
	25	+	// shutdown paths; both must record a failed, correlated end.
	26	+	for _, closeLocal := range []bool{false, true} {
	27	+	left, right := net.Pipe()
	28	+	var started <-chan struct{}
	29	+	conn := left
	30	+	if closeLocal {
	31	+	observedConn := &ipcReadStartedConn{Conn: left, started: make(chan struct{})}
	32	+	conn, started = observedConn, observedConn.started
	33	+	}
	34	+	transport := NewTransport(conn, WithRuntimeObserver(observer))
	35	+	recvDone := make(chan error, 1)
	36	+	go func() { _, err := transport.Recv(); recvDone <- err }()
	37	+	if closeLocal {
	38	+	<-started
	39	+	if err := transport.Close(); err != nil {
	40	+	t.Fatalf("Close() error = %v", err)
	41	+	}
	42	+	} else if err := right.Close(); err != nil {
	43	+	t.Fatalf("peer Close() error = %v", err)
	44	+	}
	45	+	if err := <-recvDone; err == nil {
	46	+	t.Fatal("Recv() error = nil after transport shutdown")
	47	+	}
	48	+	_ = left.Close()
	49	+	_ = right.Close()
	50	+	}
	51	+
	52	+	// A peer shutdown also makes Send fail after its start mark.
	53	+	left, right := net.Pipe()
	54	+	transport := NewTransport(left, WithRuntimeObserver(observer))
	55	+	_ = right.Close()
	56	+	if err := transport.Send(ports.Frame{Type: ports.MsgOutput, Payload: []byte("failed")}); err == nil {
	57	+	t.Fatal("Send() error = nil after peer close")
	58	+	}
	59	+	_ = transport.Close()
	60	+	if err := closer.Close(); err != nil {
	61	+	t.Fatalf("trace Close() error = %v", err)
	62	+	}
	63	+	assertIPCFailedSpanPairs(t, trace)
	64	+	}
	65	+
	66	+	type ipcReadStartedConn struct {
	67	+	net.Conn
	68	+	started chan struct{}
	69	+	once sync.Once
	70	+	}
	71	+
	72	+	func (c *ipcReadStartedConn) Read(p []byte) (int, error) {
	73	+	c.once.Do(func() { close(c.started) })
	74	+	return c.Conn.Read(p)
	75	+	}
	76	+
	77	+	func assertIPCFailedSpanPairs(t *testing.T, path string) {
	78	+	t.Helper()
	79	+	data, err := os.ReadFile(path)
	80	+	if err != nil {
	81	+	t.Fatalf("ReadFile(trace): %v", err)
	82	+	}
	83	+	type mark struct {
	84	+	ProcessID string `json:"process_id"`
	85	+	Component string `json:"component"`
	86	+	Scenario string `json:"scenario"`
	87	+	Run uint64 `json:"run"`
	88	+	Sequence uint64 `json:"sequence"`
	89	+	RequestID uint64 `json:"request_id"`
	90	+	Epoch uint64 `json:"epoch"`
	91	+	Kind string `json:"kind"`
	92	+	Valid bool `json:"valid"`
	93	+	}
	94	+	type spanKey struct {
	95	+	processID, scenario string
	96	+	run, sequence, requestID, epoch uint64
	97	+	}
	98	+	starts := make(map[spanKey]mark)
	99	+	startCounts := map[string]int{}
	100	+	for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
	101	+	var m mark
	102	+	if err := json.Unmarshal([]byte(line), &m); err != nil {
	103	+	t.Fatalf("Unmarshal(trace): %v", err)
	104	+	}
	105	+	key := spanKey{m.ProcessID, m.Scenario, m.Run, m.Sequence, m.RequestID, m.Epoch}
	106	+	switch m.Kind {
	107	+	case "adapter_receive_start", "adapter_send_start":
	108	+	if m.ProcessID != "ipc-process" \|\| m.Component != "ipc" \|\| m.Scenario != "runtime" \|\| m.Run != 1 \|\| m.Sequence == 0 \|\| m.RequestID != m.Sequence \|\| m.Epoch != m.Sequence \|\| !m.Valid {
	109	+	t.Fatalf("adapter start has invalid identity: %#v", m)
	110	+	}
	111	+	if _, exists := starts[key]; exists {
	112	+	t.Fatalf("duplicate adapter start for %#v", key)
	113	+	}
	114	+	starts[key] = m
	115	+	startCounts[m.Kind]++
	116	+	case "adapter_receive_end", "adapter_send_end":
	117	+	start, ok := starts[key]
	118	+	if !ok \|\| strings.TrimSuffix(start.Kind, "_start") != strings.TrimSuffix(m.Kind, "_end") {
	119	+	t.Fatalf("unmatched adapter end %#v", m)
	120	+	}
	121	+	if m.Valid {
	122	+	t.Fatalf("failed adapter end is valid: %#v", m)
	123	+	}
	124	+	delete(starts, key)
	125	+	}
	126	+	}
	127	+	if len(starts) != 0 {
	128	+	t.Fatalf("unmatched adapter starts: %#v", starts)
	129	+	}
	130	+	if got := startCounts["adapter_receive_start"]; got != 2 {
	131	+	t.Errorf("adapter receive starts = %d, want 2", got)
	132	+	}
	133	+	if got := startCounts["adapter_send_start"]; got != 1 {
	134	+	t.Errorf("adapter send starts = %d, want 1", got)
	135	+	}
	136	+	}
	137	+
	138	+	func TestTransportObservabilityIPCMarksCarriageWithoutChangingBytes(t *testing.T) {
	139	+	trace := t.TempDir() + "/ipc.jsonl"
	140	+	observer, closer, err := observability.NewJSONL(trace, ipcRuntimeClock{now: time.Unix(0, 211)}, "ipc-process")
	141	+	if err != nil {
	142	+	t.Fatalf("NewJSONL() error = %v", err)
	143	+	}
	144	+
	145	+	left, right := net.Pipe()
	146	+	defer func() { _ = left.Close() }()
	147	+	defer func() { _ = right.Close() }()
	148	+	client := NewTransport(left, WithRuntimeObserver(observer))
	149	+	server := NewTransport(right, WithRuntimeObserver(observer))
	150	+	want := ports.Frame{Type: ports.MsgOutput, Payload: []byte("wire bytes stay exact")}
	151	+
	152	+	var wg sync.WaitGroup
	153	+	wg.Go(func() {
	154	+	if err := client.Send(want); err != nil {
	155	+	t.Errorf("Send() error = %v", err)
	156	+	}
	157	+	})
	158	+	got, err := server.Recv()
	159	+	wg.Wait()
	160	+	if err != nil {
	161	+	t.Fatalf("Recv() error = %v", err)
	162	+	}
	163	+	if !reflect.DeepEqual(got, want) {
	164	+	t.Fatalf("carried frame = %#v, want %#v", got, want)
	165	+	}
	166	+	if err := closer.Close(); err != nil {
	167	+	t.Fatalf("trace Close() error = %v", err)
	168	+	}
	169	+	traceBytes, err := os.ReadFile(trace)
	170	+	if err != nil {
	171	+	t.Fatalf("ReadFile(trace): %v", err)
	172	+	}
	173	+	for _, kind := range []string{"adapter_send_start", "adapter_send_end", "adapter_receive_start", "adapter_receive_end"} {
	174	+	if !strings.Contains(string(traceBytes), `"kind":"`+kind+`"`) {
	175	+	t.Errorf("trace missing %s", kind)
	176	+	}
	177	+	}
	178	+	}
	179	+
	180	+	type ipcRuntimeClock struct{ now time.Time }
	181	+
	182	+	func (c ipcRuntimeClock) Now() time.Time { return c.now }
	183	+	func (c ipcRuntimeClock) NewTimer(d time.Duration) ports.Timer {
	184	+	return ipcRuntimeTimer{timer: time.NewTimer(d)}
	185	+	}
	186	+
	187	+	type ipcRuntimeTimer struct{ timer *time.Timer }
	188	+
	189	+	func (t ipcRuntimeTimer) C() <-chan time.Time { return t.timer.C }
	190	+	func (t ipcRuntimeTimer) Reset(d time.Duration) bool { return t.timer.Reset(d) }
	191	+	func (t ipcRuntimeTimer) Stop() bool { return t.timer.Stop() }

A internal/adapters/observability/jsonl.go +100 -0

@@ -0,0 +1,100 @@
	1	+	// Package observability contains concrete process-local trace sinks.
	2	+	package observability
	3	+
	4	+	import (
	5	+	"encoding/json"
	6	+	"io"
	7	+	"os"
	8	+	"sync"
	9	+
	10	+	"github.com/bnema/vev/internal/ports"
	11	+	)
	12	+
	13	+	// NewJSONL opens path as the sole timestamp owner for one OS process. Marks
	14	+	// carrying a caller timestamp are rejected rather than rewritten, preventing
	15	+	// accidental arithmetic across clock domains.
	16	+	func NewJSONL(path string, clock ports.Clock, processID string) (ports.RuntimeObserver, io.Closer, error) {
	17	+	if path == "" \|\| clock == nil \|\| processID == "" {
	18	+	return nil, nil, os.ErrInvalid
	19	+	}
	20	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_APPEND, 0o600)
	21	+	if err != nil {
	22	+	return nil, nil, err
	23	+	}
	24	+	j := &jsonl{file: f, clock: clock, processID: processID}
	25	+	return j, j, nil
	26	+	}
	27	+
	28	+	type jsonl struct {
	29	+	mu sync.Mutex
	30	+	file *os.File
	31	+	clock ports.Clock
	32	+	processID string
	33	+	closed bool
	34	+	err error
	35	+	}
	36	+
	37	+	type record struct {
	38	+	Schema uint16 `json:"schema"`
	39	+	ProcessID string `json:"process_id"`
	40	+	Component string `json:"component"`
	41	+	Scenario string `json:"scenario"`
	42	+	Run uint64 `json:"run"`
	43	+	Sequence uint64 `json:"sequence"`
	44	+	RequestID uint64 `json:"request_id"`
	45	+	Epoch uint64 `json:"epoch"`
	46	+	Kind ports.RuntimeMarkKind `json:"kind"`
	47	+	Tick int64 `json:"tick"`
	48	+	Bytes uint64 `json:"bytes"`
	49	+	Fragments uint64 `json:"fragments"`
	50	+	Retransmits uint64 `json:"retransmits"`
	51	+	Pending uint64 `json:"pending"`
	52	+	AckRTTNanos int64 `json:"ack_rtt_nanos"`
	53	+	Valid bool `json:"valid"`
	54	+	}
	55	+
	56	+	func (j *jsonl) ObserveRuntime(mark ports.RuntimeMark) {
	57	+	// Tick is a write-once value owned by this adapter. Reject it before any
	58	+	// mutation so callers cannot smuggle a timestamp from their own clock.
	59	+	if j == nil \|\| mark.Tick != 0 {
	60	+	return
	61	+	}
	62	+	mark.ProcessID = j.processID
	63	+	if !ports.RuntimeMarkValid(mark) {
	64	+	return
	65	+	}
	66	+	j.mu.Lock()
	67	+	defer j.mu.Unlock()
	68	+	if j.closed {
	69	+	return
	70	+	}
	71	+	r := record{Schema: mark.Schema, ProcessID: mark.ProcessID, Component: mark.Component, Scenario: mark.Scenario, Run: mark.Run, Sequence: mark.Sequence, RequestID: mark.RequestID, Epoch: mark.Epoch, Kind: mark.Kind, Tick: j.clock.Now().UnixNano(), Bytes: mark.Bytes, Fragments: mark.Fragments, Retransmits: mark.Retransmits, Pending: mark.Pending, AckRTTNanos: mark.AckRTTNanos, Valid: mark.Valid}
	72	+	encoded, err := json.Marshal(r)
	73	+	if err != nil {
	74	+	j.recordError(err)
	75	+	return
	76	+	}
	77	+	if _, err := j.file.Write(append(encoded, '\n')); err != nil {
	78	+	j.recordError(err)
	79	+	}
	80	+	}
	81	+
	82	+	func (j *jsonl) recordError(err error) {
	83	+	if err != nil && j.err == nil {
	84	+	j.err = err
	85	+	}
	86	+	}
	87	+
	88	+	func (j *jsonl) Close() error {
	89	+	j.mu.Lock()
	90	+	defer j.mu.Unlock()
	91	+	if j.closed {
	92	+	return nil
	93	+	}
	94	+	j.closed = true
	95	+	closeErr := j.file.Close()
	96	+	if j.err != nil {
	97	+	return j.err
	98	+	}
	99	+	return closeErr
	100	+	}

A internal/adapters/observability/jsonl_test.go +193 -0

@@ -0,0 +1,193 @@
	1	+	package observability
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/json"
	6	+	"os"
	7	+	"sync"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/vev/internal/ports"
	12	+	)
	13	+
	14	+	func TestTransportObservabilityJSONLOwnsTicksAndSchema(t *testing.T) {
	15	+	path := t.TempDir() + "/daemon.jsonl"
	16	+	observer, closer, err := NewJSONL(path, fixedRuntimeClock{now: time.Unix(0, 41)}, "daemon-process")
	17	+	if err != nil {
	18	+	t.Fatalf("NewJSONL() error = %v", err)
	19	+	}
	20	+
	21	+	observer.ObserveRuntime(validRuntimeMark(ports.RuntimeComposeStart))
	22	+	// A producer's clock domain must never enter a JSONL record. Rejection is
	23	+	// observable as no second record, rather than a silently rewritten mark.
	24	+	producerTicked := validRuntimeMark(ports.RuntimeComposeEnd)
	25	+	producerTicked.Tick = 99
	26	+	observer.ObserveRuntime(producerTicked)
	27	+	if err := closer.Close(); err != nil {
	28	+	t.Fatalf("Close() error = %v", err)
	29	+	}
	30	+
	31	+	records := readJSONLRecords(t, path)
	32	+	if len(records) != 1 {
	33	+	t.Fatalf("JSONL records = %d, want 1 (caller Tick must be rejected)", len(records))
	34	+	}
	35	+	wantKeys := map[string]bool{
	36	+	"schema": true, "process_id": true, "component": true, "scenario": true,
	37	+	"run": true, "sequence": true, "request_id": true, "epoch": true,
	38	+	"kind": true, "tick": true, "bytes": true, "fragments": true,
	39	+	"retransmits": true, "pending": true, "ack_rtt_nanos": true, "valid": true,
	40	+	}
	41	+	if len(records[0]) != len(wantKeys) {
	42	+	t.Fatalf("JSONL fields = %#v, want exactly canonical schema", records[0])
	43	+	}
	44	+	for key := range wantKeys {
	45	+	if _, ok := records[0][key]; !ok {
	46	+	t.Errorf("JSONL missing %q", key)
	47	+	}
	48	+	}
	49	+	if got := records[0]["tick"]; got != float64(41) {
	50	+	t.Errorf("JSONL tick = %v, want observer clock tick 41", got)
	51	+	}
	52	+	if got := records[0]["process_id"]; got != "daemon-process" {
	53	+	t.Errorf("JSONL process_id = %v, want observer process", got)
	54	+	}
	55	+	}
	56	+
	57	+	func TestTransportObservabilityJSONLRejectsInvalidRequiredFields(t *testing.T) {
	58	+	for _, mark := range []ports.RuntimeMark{
	59	+	{},
	60	+	{Schema: 1, ProcessID: "p", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, Kind: ports.RuntimeEmitStart},
	61	+	{Schema: 2, ProcessID: "p", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, Kind: ports.RuntimeEmitStart, Valid: true},
	62	+	} {
	63	+	path := t.TempDir() + "/invalid.jsonl"
	64	+	observer, closer, err := NewJSONL(path, fixedRuntimeClock{now: time.Unix(0, 5)}, "p")
	65	+	if err != nil {
	66	+	t.Fatalf("NewJSONL() error = %v", err)
	67	+	}
	68	+	observer.ObserveRuntime(mark)
	69	+	if err := closer.Close(); err != nil {
	70	+	t.Fatalf("Close() error = %v", err)
	71	+	}
	72	+	if records := readJSONLRecords(t, path); len(records) != 0 {
	73	+	t.Errorf("invalid mark %#v wrote %#v", mark, records)
	74	+	}
	75	+	}
	76	+	}
	77	+
	78	+	func TestTransportObservabilityJSONLWriteErrorSurfacesOnClose(t *testing.T) {
	79	+	// /dev/full accepts Open but rejects every write, exercising the concrete
	80	+	// observer lifecycle without changing the one-way RuntimeObserver port.
	81	+	observer, closer, err := NewJSONL("/dev/full", fixedRuntimeClock{now: time.Unix(0, 1)}, "p")
	82	+	if err != nil {
	83	+	t.Skipf("/dev/full unavailable: %v", err)
	84	+	}
	85	+	observer.ObserveRuntime(validRuntimeMark(ports.RuntimeEmitStart))
	86	+	if err := closer.Close(); err == nil {
	87	+	t.Fatal("Close() error = nil, want surfaced JSONL write error")
	88	+	}
	89	+	}
	90	+
	91	+	func TestTransportObservabilitySameProcessSpanClockDomains(t *testing.T) {
	92	+	for _, tc := range []struct {
	93	+	name string
	94	+	start ports.RuntimeMarkKind
	95	+	end ports.RuntimeMarkKind
	96	+	field string
	97	+	}{
	98	+	{"diff", ports.RuntimeDiffStart, ports.RuntimeDiffEnd, "diff_duration"},
	99	+	{"queue", ports.RuntimeQueueEnqueued, ports.RuntimeQueueDequeued, "queue_wait"},
	100	+	{"ack", ports.RuntimeACKBlockedStart, ports.RuntimeACKBlockedEnd, "ack_blocked_interval"},
	101	+	{"send", ports.RuntimeAdapterSendStart, ports.RuntimeAdapterSendEnd, "adapter_send_duration"},
	102	+	{"receive", ports.RuntimeAdapterReceiveStart, ports.RuntimeAdapterReceiveEnd, "adapter_receive_duration"},
	103	+	} {
	104	+	t.Run(tc.name, func(t *testing.T) {
	105	+	start := validRuntimeMark(tc.start)
	106	+	start.Tick = 11
	107	+	end := start
	108	+	end.Kind, end.Tick = tc.end, 19
	109	+	if got, err := sameProcessDuration(start, end); err != nil \|\| got != 8 {
	110	+	t.Fatalf("%s = %d, %v; want 8, nil", tc.field, got, err)
	111	+	}
	112	+
	113	+	end.ProcessID = "other-process"
	114	+	if _, err := sameProcessDuration(start, end); err == nil {
	115	+	t.Fatalf("%s accepted cross-process ticks", tc.field)
	116	+	}
	117	+	})
	118	+	}
	119	+	}
	120	+
	121	+	func TestTransportObservabilityJSONLSerializesConcurrentMarks(t *testing.T) {
	122	+	path := t.TempDir() + "/concurrent.jsonl"
	123	+	observer, closer, err := NewJSONL(path, fixedRuntimeClock{now: time.Unix(0, 73)}, "ipc-process")
	124	+	if err != nil {
	125	+	t.Fatalf("NewJSONL() error = %v", err)
	126	+	}
	127	+	var wg sync.WaitGroup
	128	+	for i := uint64(0); i < 32; i++ {
	129	+	wg.Add(1)
	130	+	go func(sequence uint64) {
	131	+	defer wg.Done()
	132	+	mark := validRuntimeMark(ports.RuntimeAdapterSendEnd)
	133	+	mark.Sequence = sequence + 1
	134	+	observer.ObserveRuntime(mark)
	135	+	}(i)
	136	+	}
	137	+	wg.Wait()
	138	+	if err := closer.Close(); err != nil {
	139	+	t.Fatalf("Close() error = %v", err)
	140	+	}
	141	+	if records := readJSONLRecords(t, path); len(records) != 32 {
	142	+	t.Fatalf("concurrently serialized records = %d, want 32", len(records))
	143	+	}
	144	+	}
	145	+
	146	+	func sameProcessDuration(start, end ports.RuntimeMark) (int64, error) {
	147	+	if start.ProcessID == "" \|\| start.ProcessID != end.ProcessID {
	148	+	return 0, os.ErrInvalid
	149	+	}
	150	+	if start.Scenario != end.Scenario \|\| start.Run != end.Run \|\| start.Sequence != end.Sequence \|\| start.RequestID != end.RequestID \|\| start.Epoch != end.Epoch \|\| end.Tick < start.Tick {
	151	+	return 0, os.ErrInvalid
	152	+	}
	153	+	return end.Tick - start.Tick, nil
	154	+	}
	155	+
	156	+	func validRuntimeMark(kind ports.RuntimeMarkKind) ports.RuntimeMark {
	157	+	return ports.RuntimeMark{Schema: 1, ProcessID: "daemon-process", Component: "daemon", Scenario: "resize", Run: 7, Sequence: 9, RequestID: 11, Epoch: 13, Kind: kind, Valid: true}
	158	+	}
	159	+
	160	+	func readJSONLRecords(t *testing.T, path string) []map[string]any {
	161	+	t.Helper()
	162	+	f, err := os.Open(path)
	163	+	if err != nil {
	164	+	t.Fatalf("Open(%q): %v", path, err)
	165	+	}
	166	+	defer func() { _ = f.Close() }()
	167	+	var records []map[string]any
	168	+	scanner := bufio.NewScanner(f)
	169	+	for scanner.Scan() {
	170	+	var record map[string]any
	171	+	if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
	172	+	t.Fatalf("invalid JSONL record %q: %v", scanner.Text(), err)
	173	+	}
	174	+	records = append(records, record)
	175	+	}
	176	+	if err := scanner.Err(); err != nil {
	177	+	t.Fatalf("reading JSONL: %v", err)
	178	+	}
	179	+	return records
	180	+	}
	181	+
	182	+	type fixedRuntimeClock struct{ now time.Time }
	183	+
	184	+	func (c fixedRuntimeClock) Now() time.Time { return c.now }
	185	+	func (c fixedRuntimeClock) NewTimer(d time.Duration) ports.Timer {
	186	+	return fixedRuntimeTimer{timer: time.NewTimer(d)}
	187	+	}
	188	+
	189	+	type fixedRuntimeTimer struct{ timer *time.Timer }
	190	+
	191	+	func (t fixedRuntimeTimer) C() <-chan time.Time { return t.timer.C }
	192	+	func (t fixedRuntimeTimer) Reset(d time.Duration) bool { return t.timer.Reset(d) }
	193	+	func (t fixedRuntimeTimer) Stop() bool { return t.timer.Stop() }

M internal/adapters/remote/remote_factory.go +17 -8

 )
 // DialerFactory selects the requested remote transport adapter.
-type DialerFactory struct{}
+type DialerFactory struct{ observer ports.RuntimeObserver }
 var _ ports.RemoteDialerFactory = DialerFactory{}
 func NewDialerFactory() DialerFactory { return DialerFactory{} }
-func (DialerFactory) DialerForRemote(target string, session string, mode ports.RemoteTransportMode, log *slog.Logger) (ports.Dialer, error) {
+// NewDialerFactoryWithRuntimeObserver wires the one process-local sink into
+// whichever concrete remote carriage adapter is selected.
+func NewDialerFactoryWithRuntimeObserver(observer ports.RuntimeObserver) DialerFactory {
+	return DialerFactory{observer: observer}
+}
+func (f DialerFactory) DialerForRemote(target string, session string, mode ports.RemoteTransportMode, log *slog.Logger) (ports.Dialer, error) {
 	switch mode {
 	case ports.RemoteTransportUDP:
 		if log != nil {
 			log.Info("remote transport selected", "mode", mode, "target", target, "session", session)
 		}
-		return dgram.NewRemoteDialerWithLogger(target, session, log), nil
+		dialer := dgram.NewRemoteDialerWithLogger(target, session, log)
+		dialer.RuntimeObserver = f.observer
+		return dialer, nil
 	case ports.RemoteTransportStdio:
 		if log != nil {
 			log.Info("remote transport selected", "mode", mode, "target", target, "session", session)
 		}
-		return stdioDialer{target: target, session: session, log: log}, nil
+		return stdioDialer{target: target, session: session, log: log, observer: f.observer}, nil
 	default:
 		return nil, fmt.Errorf("vev: unsupported remote transport %q", mode)
 	}
 }
 type stdioDialer struct {
-	target  string
+	target   string
-	session string
+	session  string
-	log     *slog.Logger
+	log      *slog.Logger
+	observer ports.RuntimeObserver
 }
 func (d stdioDialer) Dial(ctx context.Context) (ports.Transport, error) {
-	return sshstdio.DialContext(ctx, d.target, d.session, d.log)
+	return sshstdio.DialContextWithRuntimeObserver(ctx, d.target, d.session, d.log, sshstdio.WithRuntimeObserver(d.observer))
 }

M internal/adapters/sshstdio/transport.go +127 -14


@@ -34,9 +34,23 @@ var (
34		type closeFunc func() error	34		type closeFunc func() error
35		type eofErrFunc func() error	35		type eofErrFunc func() error
36			36
			37	+	type Option func(*transport)
			38	+
			39	+	// WithRuntimeObserver enables process-local transport marks; it deliberately
			40	+	// takes only an observer so a carriage adapter never owns trace time.
			41	+	func WithRuntimeObserver(observer ports.RuntimeObserver) Option {
			42	+	return func(t *transport) { t.observer = observer }
			43	+	}
			44	+
37		// NewTransport wraps separate reader/writer streams as a framed Transport.	45		// NewTransport wraps separate reader/writer streams as a framed Transport.
38	-	func NewTransport(r io.Reader, w io.Writer, closeFn closeFunc) ports.Transport {	46	+	func NewTransport(r io.Reader, w io.Writer, closeFn closeFunc, opts ...Option) ports.Transport {
39	-	return newTransport(r, w, closeFn, nil)	47	+	t := newTransport(r, w, closeFn, nil).(*transport)
			48	+	for _, opt := range opts {
			49	+	if opt != nil {
			50	+	opt(t)
			51	+	}
			52	+	}
			53	+	return t
40		}	54		}
41			55
42		func newTransport(r io.Reader, w io.Writer, closeFn closeFunc, eofErr eofErrFunc) ports.Transport {	56		func newTransport(r io.Reader, w io.Writer, closeFn closeFunc, eofErr eofErrFunc) ports.Transport {

@@ -47,18 +61,25 @@ func newTransport(r io.Reader, w io.Writer, closeFn closeFunc, eofErr eofErrFunc
47		}	61		}
48			62
49		type transport struct {	63		type transport struct {
50	-	r io.Reader	64	+	r io.Reader
51	-	w io.Writer	65	+	w io.Writer
52	-	close closeFunc	66	+	close closeFunc
53	-	eofErr eofErrFunc	67	+	eofErr eofErrFunc
			68	+	observer ports.RuntimeObserver
			69	+	operationMu sync.Mutex
			70	+	operationCount int
			71	+	closing bool
			72	+	operationsDone chan struct{}
54			73
55		mu sync.Mutex	74		mu sync.Mutex
56		readBuf []byte	75		readBuf []byte
57		}	76		}
58			77
59		func (t *transport) Send(f ports.Frame) error {	78		func (t *transport) Send(f ports.Frame) error {
			79	+	end := t.beginOperation(ports.RuntimeAdapterSendStart, uint64(len(f.Payload)))
60		n := 1 + len(f.Payload)	80		n := 1 + len(f.Payload)
61		if n > maxFrameLen {	81		if n > maxFrameLen {
			82	+	end(false)
62		return ErrFrameTooLarge	83		return ErrFrameTooLarge
63		}	84		}
64			85

@@ -68,22 +89,28 @@ func (t *transport) Send(f ports.Frame) error {
68		copy(buf[frameHeaderLen+1:], f.Payload)	89		copy(buf[frameHeaderLen+1:], f.Payload)
69			90
70		t.mu.Lock()	91		t.mu.Lock()
71	-	defer t.mu.Unlock()
72		_, err := t.w.Write(buf)	92		_, err := t.w.Write(buf)
			93	+	t.mu.Unlock()
			94	+	end(err == nil)
73		return err	95		return err
74		}	96		}
75			97
76		func (t *transport) Recv() (ports.Frame, error) {	98		func (t *transport) Recv() (ports.Frame, error) {
			99	+	end := t.beginOperation(ports.RuntimeAdapterReceiveStart, 0)
77		var hdr [frameHeaderLen]byte	100		var hdr [frameHeaderLen]byte
78		if _, err := io.ReadFull(t.r, hdr[:]); err != nil {	101		if _, err := io.ReadFull(t.r, hdr[:]); err != nil {
79	-	return ports.Frame{}, t.mapEOFError(err)	102	+	err = t.mapEOFError(err)
			103	+	end(false)
			104	+	return ports.Frame{}, err
80		}	105		}
81			106
82		n := binary.BigEndian.Uint32(hdr[:])	107		n := binary.BigEndian.Uint32(hdr[:])
83		if n == 0 {	108		if n == 0 {
			109	+	end(false)
84		return ports.Frame{}, ErrZeroLengthFrame	110		return ports.Frame{}, ErrZeroLengthFrame
85		}	111		}
86		if n > maxFrameLen {	112		if n > maxFrameLen {
			113	+	end(false)
87		return ports.Frame{}, ErrFrameTooLarge	114		return ports.Frame{}, ErrFrameTooLarge
88		}	115		}
89			116

@@ -93,13 +120,35 @@ func (t *transport) Recv() (ports.Frame, error) {
93		t.readBuf = t.readBuf[:n]	120		t.readBuf = t.readBuf[:n]
94		}	121		}
95		if _, err := io.ReadFull(t.r, t.readBuf); err != nil {	122		if _, err := io.ReadFull(t.r, t.readBuf); err != nil {
96	-	return ports.Frame{}, t.mapEOFError(err)	123	+	err = t.mapEOFError(err)
			124	+	end(false)
			125	+	return ports.Frame{}, err
97		}	126		}
98			127
99		payload := append([]byte(nil), t.readBuf[1:]...)	128		payload := append([]byte(nil), t.readBuf[1:]...)
			129	+	end(true)
100		return ports.Frame{Type: ports.MsgType(t.readBuf[0]), Payload: payload}, nil	130		return ports.Frame{Type: ports.MsgType(t.readBuf[0]), Payload: payload}, nil
101		}	131		}
102			132
			133	+	func (t *transport) beginOperation(start ports.RuntimeMarkKind, bytes uint64) func(bool) {
			134	+	if t.observer == nil {
			135	+	return func(bool) {}
			136	+	}
			137	+	if !t.beginObservedOperation() {
			138	+	return func(bool) {}
			139	+	}
			140	+	correlation := ports.NewRuntimeCorrelation()
			141	+	t.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("sshstdio", correlation, start, bytes, true))
			142	+	end := ports.RuntimeAdapterSendEnd
			143	+	if start == ports.RuntimeAdapterReceiveStart {
			144	+	end = ports.RuntimeAdapterReceiveEnd
			145	+	}
			146	+	return func(valid bool) {
			147	+	defer t.finishObservedOperation()
			148	+	t.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("sshstdio", correlation, end, bytes, valid))
			149	+	}
			150	+	}
			151	+
103		func (t *transport) mapEOFError(err error) error {	152		func (t *transport) mapEOFError(err error) error {
104		if t.eofErr == nil \|\| (!errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF)) {	153		if t.eofErr == nil \|\| (!errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF)) {
105		return err	154		return err

@@ -110,7 +159,56 @@ func (t *transport) mapEOFError(err error) error {
110		return err	159		return err
111		}	160		}
112			161
113	-	func (t *transport) Close() error { return t.close() }	162	+	func (t *transport) Close() error {
			163	+	done := t.beginShutdown()
			164	+	// Closing the reader first releases a Recv blocked in io.ReadFull even if
			165	+	// the subprocess shutdown callback reports an error. This lets Recv emit
			166	+	// its failed end mark before the process observer is closed.
			167	+	var readErr error
			168	+	if r, ok := t.r.(io.Closer); ok {
			169	+	readErr = r.Close()
			170	+	}
			171	+	closeErr := t.close()
			172	+	if done != nil {
			173	+	<-done
			174	+	}
			175	+	if closeErr != nil {
			176	+	return closeErr
			177	+	}
			178	+	return readErr
			179	+	}
			180	+
			181	+	func (t *transport) beginObservedOperation() bool {
			182	+	t.operationMu.Lock()
			183	+	defer t.operationMu.Unlock()
			184	+	if t.closing {
			185	+	return false
			186	+	}
			187	+	if t.operationCount == 0 {
			188	+	t.operationsDone = make(chan struct{})
			189	+	}
			190	+	t.operationCount++
			191	+	return true
			192	+	}
			193	+
			194	+	func (t *transport) finishObservedOperation() {
			195	+	t.operationMu.Lock()
			196	+	defer t.operationMu.Unlock()
			197	+	t.operationCount--
			198	+	if t.operationCount == 0 {
			199	+	close(t.operationsDone)
			200	+	}
			201	+	}
			202	+
			203	+	func (t *transport) beginShutdown() <-chan struct{} {
			204	+	t.operationMu.Lock()
			205	+	defer t.operationMu.Unlock()
			206	+	t.closing = true
			207	+	if t.operationCount == 0 {
			208	+	return nil
			209	+	}
			210	+	return t.operationsDone
			211	+	}
114			212
115		type processWaiter struct {	213		type processWaiter struct {
116		cmd *exec.Cmd	214		cmd *exec.Cmd

@@ -222,18 +320,27 @@ func Dial(target, session string) (ports.Transport, error) {
222		return DialContext(context.Background(), target, session)	320		return DialContext(context.Background(), target, session)
223		}	321		}
224			322
			323	+	// DialContextWithRuntimeObserver is the option-bearing production constructor.
			324	+	func DialContextWithRuntimeObserver(ctx context.Context, target, session string, logger *slog.Logger, opts ...Option) (ports.Transport, error) {
			325	+	return dialContext(ctx, target, session, logger, opts...)
			326	+	}
			327	+
225		// DialContext is like Dial, but the context is propagated to ssh startup so a	328		// DialContext is like Dial, but the context is propagated to ssh startup so a
226		// canceled attach attempt interrupts the local ssh process. Callers may pass a	329		// canceled attach attempt interrupts the local ssh process. Callers may pass a
227		// logger to record ssh start failures and non-clean exits without logging the	330		// logger to record ssh start failures and non-clean exits without logging the
228		// generated command line.	331		// generated command line.
229		func DialContext(ctx context.Context, target, session string, logger ...*slog.Logger) (ports.Transport, error) {	332		func DialContext(ctx context.Context, target, session string, logger ...*slog.Logger) (ports.Transport, error) {
230	-	if err := ctx.Err(); err != nil {
231	-	return nil, err
232	-	}
233		var log *slog.Logger	333		var log *slog.Logger
234		if len(logger) > 0 {	334		if len(logger) > 0 {
235		log = logger[0]	335		log = logger[0]
236		}	336		}
			337	+	return dialContext(ctx, target, session, log)
			338	+	}
			339	+
			340	+	func dialContext(ctx context.Context, target, session string, log *slog.Logger, opts ...Option) (ports.Transport, error) {
			341	+	if err := ctx.Err(); err != nil {
			342	+	return nil, err
			343	+	}
237		spec := BuildCommand(target, session)	344		spec := BuildCommand(target, session)
238		cmd := exec.CommandContext(ctx, spec.Path, spec.Args...)	345		cmd := exec.CommandContext(ctx, spec.Path, spec.Args...)
239		stdin, err := cmd.StdinPipe()	346		stdin, err := cmd.StdinPipe()

@@ -254,5 +361,11 @@ func DialContext(ctx context.Context, target, session string, logger ...*slog.Lo
254		}	361		}
255			362
256		waiter := newProcessWaiter(cmd, stdin, &stderr, sshCloseTimeout, log, target, session)	363		waiter := newProcessWaiter(cmd, stdin, &stderr, sshCloseTimeout, log, target, session)
257	-	return newTransport(stdout, stdin, waiter.close, waiter.eofErr), nil	364	+	transport := newTransport(stdout, stdin, waiter.close, waiter.eofErr).(*transport)
			365	+	for _, opt := range opts {
			366	+	if opt != nil {
			367	+	opt(transport)
			368	+	}
			369	+	}
			370	+	return transport, nil
258		}	371		}

A internal/adapters/sshstdio/transport_observability_test.go +173 -0

@@ -0,0 +1,173 @@
	1	+	package sshstdio
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"os"
	8	+	"strings"
	9	+	"sync"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"github.com/bnema/vev/internal/adapters/observability"
	14	+	"github.com/bnema/vev/internal/ports"
	15	+	)
	16	+
	17	+	func TestTransportObservabilitySSHStdioEOFEndsReceive(t *testing.T) {
	18	+	tracePath := t.TempDir() + "/ssh-eof.jsonl"
	19	+	observer, closer, err := observability.NewJSONL(tracePath, sshRuntimeClock{now: time.Unix(0, 309)}, "ssh-process")
	20	+	if err != nil {
	21	+	t.Fatalf("NewJSONL() error = %v", err)
	22	+	}
	23	+	if _, err := NewTransport(bytes.NewReader(nil), &bytes.Buffer{}, nil, WithRuntimeObserver(observer)).Recv(); err == nil {
	24	+	t.Fatal("Recv() error = nil at EOF")
	25	+	}
	26	+	if err := closer.Close(); err != nil {
	27	+	t.Fatalf("trace Close() error = %v", err)
	28	+	}
	29	+	assertSSHReceiveFailurePair(t, tracePath)
	30	+	}
	31	+
	32	+	func TestTransportObservabilitySSHStdioCloseEndsBlockedReceive(t *testing.T) {
	33	+	tracePath := t.TempDir() + "/ssh-close.jsonl"
	34	+	observer, closer, err := observability.NewJSONL(tracePath, sshRuntimeClock{now: time.Unix(0, 311)}, "ssh-process")
	35	+	if err != nil {
	36	+	t.Fatalf("NewJSONL() error = %v", err)
	37	+	}
	38	+	reader := &sshShutdownReader{entered: make(chan struct{}), done: make(chan struct{})}
	39	+	shutdownErr := errors.New("ssh shutdown failed")
	40	+	transport := NewTransport(reader, &bytes.Buffer{}, func() error { return shutdownErr }, WithRuntimeObserver(observer))
	41	+	recvDone := make(chan error, 1)
	42	+	go func() { _, err := transport.Recv(); recvDone <- err }()
	43	+	<-reader.entered
	44	+	if err := transport.Close(); !errors.Is(err, shutdownErr) {
	45	+	t.Fatalf("Close() error = %v, want %v", err, shutdownErr)
	46	+	}
	47	+	if err := <-recvDone; err == nil {
	48	+	t.Fatal("Recv() error = nil after Close()")
	49	+	}
	50	+	if err := closer.Close(); err != nil {
	51	+	t.Fatalf("trace Close() error = %v", err)
	52	+	}
	53	+	assertSSHReceiveFailurePair(t, tracePath)
	54	+	}
	55	+
	56	+	type sshShutdownReader struct {
	57	+	entered chan struct{}
	58	+	done chan struct{}
	59	+	once sync.Once
	60	+	}
	61	+
	62	+	func (r *sshShutdownReader) Read([]byte) (int, error) {
	63	+	r.once.Do(func() { close(r.entered) })
	64	+	<-r.done
	65	+	return 0, errors.New("reader closed")
	66	+	}
	67	+
	68	+	func (r *sshShutdownReader) Close() error {
	69	+	r.once.Do(func() { close(r.entered) })
	70	+	close(r.done)
	71	+	return nil
	72	+	}
	73	+
	74	+	func assertSSHReceiveFailurePair(t *testing.T, path string) {
	75	+	t.Helper()
	76	+	data, err := os.ReadFile(path)
	77	+	if err != nil {
	78	+	t.Fatalf("ReadFile(trace): %v", err)
	79	+	}
	80	+	var starts, ends []struct {
	81	+	ProcessID string `json:"process_id"`
	82	+	Scenario string `json:"scenario"`
	83	+	Run uint64 `json:"run"`
	84	+	Sequence uint64 `json:"sequence"`
	85	+	RequestID uint64 `json:"request_id"`
	86	+	Epoch uint64 `json:"epoch"`
	87	+	Kind string `json:"kind"`
	88	+	Valid bool `json:"valid"`
	89	+	}
	90	+	for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
	91	+	var mark struct {
	92	+	ProcessID string `json:"process_id"`
	93	+	Scenario string `json:"scenario"`
	94	+	Run uint64 `json:"run"`
	95	+	Sequence uint64 `json:"sequence"`
	96	+	RequestID uint64 `json:"request_id"`
	97	+	Epoch uint64 `json:"epoch"`
	98	+	Kind string `json:"kind"`
	99	+	Valid bool `json:"valid"`
	100	+	}
	101	+	if err := json.Unmarshal([]byte(line), &mark); err != nil {
	102	+	t.Fatalf("Unmarshal(trace): %v", err)
	103	+	}
	104	+	switch mark.Kind {
	105	+	case "adapter_receive_start":
	106	+	starts = append(starts, mark)
	107	+	case "adapter_receive_end":
	108	+	ends = append(ends, mark)
	109	+	}
	110	+	}
	111	+	if len(starts) != 1 \|\| len(ends) != 1 {
	112	+	t.Fatalf("receive spans start=%d end=%d, want exactly one pair", len(starts), len(ends))
	113	+	}
	114	+	start, end := starts[0], ends[0]
	115	+	if start.ProcessID != end.ProcessID \|\| start.Scenario != end.Scenario \|\| start.Run != end.Run \|\| start.Sequence != end.Sequence \|\| start.RequestID != end.RequestID \|\| start.Epoch != end.Epoch {
	116	+	t.Fatalf("mismatched receive correlation: start=%+v end=%+v", start, end)
	117	+	}
	118	+	if !start.Valid \|\| end.Valid {
	119	+	t.Fatalf("receive validity start=%t end=%t, want true/false", start.Valid, end.Valid)
	120	+	}
	121	+	}
	122	+
	123	+	func TestTransportObservabilitySSHStdioPreservesCarriage(t *testing.T) {
	124	+	tracePath := t.TempDir() + "/ssh.jsonl"
	125	+	observer, closer, err := observability.NewJSONL(tracePath, sshRuntimeClock{now: time.Unix(0, 307)}, "ssh-process")
	126	+	if err != nil {
	127	+	t.Fatalf("NewJSONL() error = %v", err)
	128	+	}
	129	+
	130	+	frame := ports.Frame{Type: ports.MsgOutput, Payload: []byte("stdio bytes stay exact")}
	131	+	var observed, baseline bytes.Buffer
	132	+	if err := NewTransport(bytes.NewReader(nil), &observed, nil, WithRuntimeObserver(observer)).Send(frame); err != nil {
	133	+	t.Fatalf("observed Send() error = %v", err)
	134	+	}
	135	+	if err := NewTransport(bytes.NewReader(nil), &baseline, nil).Send(frame); err != nil {
	136	+	t.Fatalf("baseline Send() error = %v", err)
	137	+	}
	138	+	if !bytes.Equal(observed.Bytes(), baseline.Bytes()) {
	139	+	t.Fatalf("observer changed SSH stdio wire bytes: got %x, want %x", observed.Bytes(), baseline.Bytes())
	140	+	}
	141	+	received, err := NewTransport(bytes.NewReader(baseline.Bytes()), &bytes.Buffer{}, nil, WithRuntimeObserver(observer)).Recv()
	142	+	if err != nil {
	143	+	t.Fatalf("observed Recv() error = %v", err)
	144	+	}
	145	+	if received.Type != frame.Type \|\| !bytes.Equal(received.Payload, frame.Payload) {
	146	+	t.Fatalf("observed Recv() = %#v, want %#v", received, frame)
	147	+	}
	148	+	if err := closer.Close(); err != nil {
	149	+	t.Fatalf("trace Close() error = %v", err)
	150	+	}
	151	+	trace, err := os.ReadFile(tracePath)
	152	+	if err != nil {
	153	+	t.Fatalf("ReadFile(trace): %v", err)
	154	+	}
	155	+	for _, kind := range []string{"adapter_send_start", "adapter_send_end", "adapter_receive_start", "adapter_receive_end"} {
	156	+	if !strings.Contains(string(trace), `"kind":"`+kind+`"`) {
	157	+	t.Errorf("trace missing %s", kind)
	158	+	}
	159	+	}
	160	+	}
	161	+
	162	+	type sshRuntimeClock struct{ now time.Time }
	163	+
	164	+	func (c sshRuntimeClock) Now() time.Time { return c.now }
	165	+	func (c sshRuntimeClock) NewTimer(d time.Duration) ports.Timer {
	166	+	return sshRuntimeTimer{timer: time.NewTimer(d)}
	167	+	}
	168	+
	169	+	type sshRuntimeTimer struct{ timer *time.Timer }
	170	+
	171	+	func (t sshRuntimeTimer) C() <-chan time.Time { return t.timer.C }
	172	+	func (t sshRuntimeTimer) Reset(d time.Duration) bool { return t.timer.Reset(d) }
	173	+	func (t sshRuntimeTimer) Stop() bool { return t.timer.Stop() }

M internal/app/daemon_integration_test.go +5 -5


@@ -74,7 +74,7 @@ func recvPump(tr ports.Transport) *pump {
74		// attach dials, handshakes, and returns the transport plus its frame pump.	74		// attach dials, handshakes, and returns the transport plus its frame pump.
75		func attach(t testing.T, dir string, intent uint8, name string, sz domain.Size) (ports.Transport, pump) {	75		func attach(t testing.T, dir string, intent uint8, name string, sz domain.Size) (ports.Transport, pump) {
76		t.Helper()	76		t.Helper()
77	-	tr, err := ipc.Dial(dir)	77	+	tr, err := ipc.DialContext(context.Background(), dir)
78		require.NoError(t, err)	78		require.NoError(t, err)
79		hello := ports.Hello{Version: ports.ProtocolVersion, Intent: intent, Name: name, Size: sz, TermEnv: "xterm-256color"}	79		hello := ports.Hello{Version: ports.ProtocolVersion, Intent: intent, Name: name, Size: sz, TermEnv: "xterm-256color"}
80		require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgHello, Payload: ports.MarshalHello(hello)}))	80		require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgHello, Payload: ports.MarshalHello(hello)}))

@@ -91,7 +91,7 @@ func attach(t *testing.T, dir string, intent uint8, name string, sz domain.Size)
91			91
92		func listRemoteSessions(t *testing.T, dir string) ports.Sessions {	92		func listRemoteSessions(t *testing.T, dir string) ports.Sessions {
93		t.Helper()	93		t.Helper()
94	-	tr, err := ipc.Dial(dir)	94	+	tr, err := ipc.DialContext(context.Background(), dir)
95		require.NoError(t, err)	95		require.NoError(t, err)
96		defer func() { _ = tr.Close() }()	96		defer func() { _ = tr.Close() }()
97		require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgList, Payload: ports.MarshalList(ports.List{})}))	97		require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgList, Payload: ports.MarshalList(ports.List{})}))

@@ -104,7 +104,7 @@ func listRemoteSessions(t *testing.T, dir string) ports.Sessions {
104		}	104		}
105			105
106		func killAll(dir string) error {	106		func killAll(dir string) error {
107	-	tr, err := ipc.Dial(dir)	107	+	tr, err := ipc.DialContext(context.Background(), dir)
108		if err != nil {	108		if err != nil {
109		return err	109		return err
110		}	110		}

@@ -248,7 +248,7 @@ func TestIntegration_CommandPaletteRenamesEphemeralSession(t *testing.T) {
248		text = awaitScreenText(t, p, sz, " work ")	248		text = awaitScreenText(t, p, sz, " work ")
249		require.NotContains(t, text, "work*")	249		require.NotContains(t, text, "work*")
250			250
251	-	listTr, err := ipc.Dial(dir)	251	+	listTr, err := ipc.DialContext(context.Background(), dir)
252		require.NoError(t, err)	252		require.NoError(t, err)
253		defer func() { _ = listTr.Close() }()	253		defer func() { _ = listTr.Close() }()
254		require.NoError(t, listTr.Send(ports.Frame{Type: ports.MsgList, Payload: ports.MarshalList(ports.List{})}))	254		require.NoError(t, listTr.Send(ports.Frame{Type: ports.MsgList, Payload: ports.MarshalList(ports.List{})}))

@@ -367,7 +367,7 @@ func TestIntegration_KillAllShutsDownDaemon(t *testing.T) {
367		tr2, _ := attach(t, dir, ports.IntentNew, "two", sz)	367		tr2, _ := attach(t, dir, ports.IntentNew, "two", sz)
368		require.NoError(t, tr2.Close())	368		require.NoError(t, tr2.Close())
369			369
370	-	killTr, err := ipc.Dial(dir)	370	+	killTr, err := ipc.DialContext(context.Background(), dir)
371		require.NoError(t, err)	371		require.NoError(t, err)
372		defer func() { _ = killTr.Close() }()	372		defer func() { _ = killTr.Close() }()
373		require.NoError(t, killTr.Send(ports.Frame{Type: ports.MsgKill, Payload: ports.MarshalKill(ports.Kill{All: true})}))	373		require.NoError(t, killTr.Send(ports.Frame{Type: ports.MsgKill, Payload: ports.MarshalKill(ports.Kill{All: true})}))

M internal/app/run.go +137 -19


@@ -22,6 +22,7 @@ import (
22		"path/filepath"	22		"path/filepath"
23		"strconv"	23		"strconv"
24		"strings"	24		"strings"
			25	+	"sync"
25		"syscall"	26		"syscall"
26		"text/tabwriter"	27		"text/tabwriter"
27		"time"	28		"time"

@@ -31,6 +32,7 @@ import (
31		"github.com/bnema/vev/internal/adapters/config"	32		"github.com/bnema/vev/internal/adapters/config"
32		"github.com/bnema/vev/internal/adapters/dgram"	33		"github.com/bnema/vev/internal/adapters/dgram"
33		"github.com/bnema/vev/internal/adapters/ipc"	34		"github.com/bnema/vev/internal/adapters/ipc"
			35	+	"github.com/bnema/vev/internal/adapters/observability"
34		"github.com/bnema/vev/internal/adapters/pty"	36		"github.com/bnema/vev/internal/adapters/pty"
35		remoteadapter "github.com/bnema/vev/internal/adapters/remote"	37		remoteadapter "github.com/bnema/vev/internal/adapters/remote"
36		"github.com/bnema/vev/internal/adapters/shellcmd"	38		"github.com/bnema/vev/internal/adapters/shellcmd"

@@ -238,6 +240,72 @@ func dispatch(ctx context.Context, cmd command) error {
238		}	240		}
239		}	241		}
240			242
			243	+	// performanceTrace creates one serialized timestamp owner for this process.
			244	+	// An empty trace environment leaves all production behavior and wire bytes
			245	+	// unchanged.
			246	+	func performanceTrace(clk ports.Clock) (ports.RuntimeObserver, io.Closer, error) {
			247	+	path, processID := os.Getenv("VEV_PERF_TRACE"), os.Getenv("VEV_PERF_PROCESS_ID")
			248	+	if path == "" {
			249	+	return nil, nil, nil
			250	+	}
			251	+	observer, closer, err := observability.NewJSONL(path, clk, processID)
			252	+	if err != nil {
			253	+	return nil, nil, err
			254	+	}
			255	+	// The harness supplies these manifest fields to every launched role. Keep a
			256	+	// valid standalone trace for operators that set only the original trace
			257	+	// variables, while ensuring harness traces match their process mapping.
			258	+	inputs := ports.RuntimeCorrelationInputs{Scenario: os.Getenv("VEV_PERF_SCENARIO"), Run: 1}
			259	+	if inputs.Scenario == "" {
			260	+	inputs.Scenario = "runtime"
			261	+	}
			262	+	if rawRun := os.Getenv("VEV_PERF_RUN"); rawRun != "" {
			263	+	inputs.Run, err = strconv.ParseUint(rawRun, 10, 64)
			264	+	if err != nil \|\| inputs.Run == 0 {
			265	+	_ = closer.Close()
			266	+	return nil, nil, fmt.Errorf("invalid VEV_PERF_RUN %q", rawRun)
			267	+	}
			268	+	}
			269	+	observer, err = ports.NewRuntimeCorrelationObserver(observer, inputs)
			270	+	if err != nil {
			271	+	_ = closer.Close()
			272	+	return nil, nil, err
			273	+	}
			274	+	reporter := ports.NewSerializedRuntimeObserver(observer, runtimeTraceQueueDepth)
			275	+	return reporter, &runtimeTraceCloser{reporter: reporter, closer: closer}, nil
			276	+	}
			277	+
			278	+	// runtimeTraceQueueDepth bounds all process-local trace producer handoffs.
			279	+	// A full queue emits the serialized diagnostic gap rather than delaying a
			280	+	// terminal, transport, or ACK progress path.
			281	+	const runtimeTraceQueueDepth = 256
			282	+
			283	+	// runtimeTraceCloser owns the two-stage trace shutdown: drain the reporter
			284	+	// before closing the concrete timestamp/file owner. sync.Once makes every
			285	+	// deferred and explicit close path safe without closing shared workers twice.
			286	+	type runtimeTraceCloser struct {
			287	+	reporter ports.SerializedRuntimeObserver
			288	+	closer io.Closer
			289	+	once sync.Once
			290	+	err error
			291	+	}
			292	+
			293	+	func (c *runtimeTraceCloser) Close() error {
			294	+	if c == nil {
			295	+	return nil
			296	+	}
			297	+	c.once.Do(func() {
			298	+	if c.reporter != nil {
			299	+	c.reporter.Flush()
			300	+	c.reporter.Close()
			301	+	}
			302	+	if c.closer != nil {
			303	+	c.err = c.closer.Close()
			304	+	}
			305	+	})
			306	+	return c.err
			307	+	}
			308	+
241		func configureLogging(component logging.Component, rotateAtRuntime bool) (*slog.Logger, io.Closer, error) {	309		func configureLogging(component logging.Component, rotateAtRuntime bool) (*slog.Logger, io.Closer, error) {
242		return logging.Setup(logging.Config{	310		return logging.Setup(logging.Config{
243		Dir: platform.StateDir(),	311		Dir: platform.StateDir(),

@@ -285,7 +353,16 @@ func runDaemon() error {
285		}	353		}
286		defer func() { _ = logCloser.Close() }()	354		defer func() { _ = logCloser.Close() }()
287			355
288	-	ln, err := ipc.Listen(ipc.SocketDir())	356	+	clk := clock.New()
			357	+	observer, observerCloser, err := performanceTrace(clk)
			358	+	if err != nil {
			359	+	return fmt.Errorf("vev: performance trace: %w", err)
			360	+	}
			361	+	if observerCloser != nil {
			362	+	defer func() { _ = observerCloser.Close() }()
			363	+	}
			364	+	// Every accepted local connection shares this process's serialized sink.
			365	+	ln, err := ipc.Listen(ipc.SocketDir(), ipc.WithRuntimeObserver(observer))
289		if err != nil {	366		if err != nil {
290		log.Error("daemon listen failed", "socket_dir", ipc.SocketDir(), "err", err)	367		log.Error("daemon listen failed", "socket_dir", ipc.SocketDir(), "err", err)
291		return fmt.Errorf("vev: daemon listen: %w", err)	368		return fmt.Errorf("vev: daemon listen: %w", err)

@@ -309,6 +386,9 @@ func runDaemon() error {
309			386
310		log.Info("daemon starting", "socket", ln.Addr())	387		log.Info("daemon starting", "socket", ln.Addr())
311		daemonOpts := []daemon.Option(nil)	388		daemonOpts := []daemon.Option(nil)
			389	+	if observer != nil {
			390	+	daemonOpts = append(daemonOpts, daemon.WithRuntimeObserver(observer))
			391	+	}
312		configPath := platform.ConfigPath()	392		configPath := platform.ConfigPath()
313		cfg, warnings, err := config.Load(configPath)	393		cfg, warnings, err := config.Load(configPath)
314		if err != nil {	394		if err != nil {

@@ -327,7 +407,6 @@ func runDaemon() error {
327		log.Info("session persistence enabled", "path", storePath)	407		log.Info("session persistence enabled", "path", storePath)
328		daemonOpts = append(daemonOpts, daemon.WithStore(store))	408		daemonOpts = append(daemonOpts, daemon.WithStore(store))
329		}	409		}
330	-	clk := clock.New()
331		d := daemon.New(pty.NewFactory(), clk, log, daemonOpts...)	410		d := daemon.New(pty.NewFactory(), clk, log, daemonOpts...)
332		watchCtx, stopWatch := context.WithCancel(ctx)	411		watchCtx, stopWatch := context.WithCancel(ctx)
333		defer stopWatch()	412		defer stopWatch()

@@ -357,13 +436,25 @@ func runAttach(ctx context.Context, intent uint8, name, remoteTarget string) err
357		}	436		}
358		defer func() { _ = logCloser.Close() }()	437		defer func() { _ = logCloser.Close() }()
359			438
			439	+	clk := clock.New()
			440	+	observer, observerCloser, err := performanceTrace(clk)
			441	+	if err != nil {
			442	+	return fmt.Errorf("vev: performance trace: %w", err)
			443	+	}
			444	+	if observerCloser != nil {
			445	+	defer func() { _ = observerCloser.Close() }()
			446	+	}
360		return runAttachWithDeps(ctx, intent, name, remoteTarget, os.Getenv("VEV"), log, runAttachDeps{	447		return runAttachWithDeps(ctx, intent, name, remoteTarget, os.Getenv("VEV"), log, runAttachDeps{
361	-	localDialer: defaultLocalDialer,	448	+	localDialer: func() ports.Dialer {
362	-	remoteDialerFactory: defaultRemoteDialerFactory(),	449	+	return localDaemonDialer{dir: ipc.SocketDir(), observer: observer}
			450	+	},
			451	+	remoteDialerFactory: remoteadapter.NewDialerFactoryWithRuntimeObserver(observer),
363		selectedRemoteTransport: os.Getenv(envRemoteTransport),	452		selectedRemoteTransport: os.Getenv(envRemoteTransport),
364	-	runClient: client.Run,	453	+	runClient: func(ctx context.Context, dialer ports.Dialer, terminal ports.Terminal, behaviorClock ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger) error {
365	-	createDetached: createDetachedLocalSession,	454	+	return client.RunWithRuntimeObserver(ctx, dialer, terminal, behaviorClock, intent, name, remote, clipboard, log, observer)
366	-	clipboard: clipboard.New(),	455	+	},
			456	+	createDetached: createDetachedLocalSession,
			457	+	clipboard: clipboard.New(),
367		})	458		})
368		}	459		}
369			460

@@ -499,10 +590,19 @@ func isDaemonVersionDrift(err error) bool {
499		return protocolErr.Code == ports.ErrVersionMismatch \|\| protocolErr.Text == legacyMalformedHelloSignal	590		return protocolErr.Code == ports.ErrVersionMismatch \|\| protocolErr.Text == legacyMalformedHelloSignal
500		}	591		}
501			592
502	-	type localDaemonDialer struct{ dir string }	593	+	type localDaemonDialer struct {
			594	+	dir string
			595	+	observer ports.RuntimeObserver
			596	+	}
503			597
504		func (d localDaemonDialer) Dial(ctx context.Context) (ports.Transport, error) {	598		func (d localDaemonDialer) Dial(ctx context.Context) (ports.Transport, error) {
505	-	return ensureDaemon(ctx, d.dir, realDial, realSpawn, defaultBackoff)	599	+	dial := realDial
			600	+	if d.observer != nil {
			601	+	dial = func(ctx context.Context, dir string) (ports.Transport, error) {
			602	+	return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(d.observer))
			603	+	}
			604	+	}
			605	+	return ensureDaemon(ctx, d.dir, dial, realSpawn, defaultBackoff)
506		}	606		}
507			607
508		func detachedLocalHello(name, cwd string) ports.Hello {	608		func detachedLocalHello(name, cwd string) ports.Hello {

@@ -584,7 +684,7 @@ func requestDaemonStop(ctx context.Context) error {
584			684
585		done := make(chan error, 1)	685		done := make(chan error, 1)
586		go func() {	686		go func() {
587	-	transport, err := realDial(ipc.SocketDir())	687	+	transport, err := realDial(ctx, ipc.SocketDir())
588		if err != nil {	688		if err != nil {
589		done <- fmt.Errorf("vev: no daemon running")	689		done <- fmt.Errorf("vev: no daemon running")
590		return	690		return

@@ -629,13 +729,22 @@ func runStdio(ctx context.Context) error {
629		defer func() { _ = logCloser.Close() }()	729		defer func() { _ = logCloser.Close() }()
630		log.Debug("stdio proxy starting")	730		log.Debug("stdio proxy starting")
631			731
632	-	transport, err := ensureDaemon(ctx, ipc.SocketDir(), realDial, realSpawn, defaultBackoff)	732	+	clk := clock.New()
			733	+	observer, observerCloser, err := performanceTrace(clk)
			734	+	if err != nil {
			735	+	return fmt.Errorf("vev: performance trace: %w", err)
			736	+	}
			737	+	if observerCloser != nil {
			738	+	defer func() { _ = observerCloser.Close() }()
			739	+	}
			740	+	transport, err := ensureDaemon(ctx, ipc.SocketDir(), func(ctx context.Context, dir string) (ports.Transport, error) {
			741	+	return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(observer))
			742	+	}, realSpawn, defaultBackoff)
633		if err != nil {	743		if err != nil {
634		return err	744		return err
635		}	745		}
636		defer func() { _ = transport.Close() }()	746		defer func() { _ = transport.Close() }()
637	-		747	+	stdio := sshstdio.NewTransport(os.Stdin, os.Stdout, nil, sshstdio.WithRuntimeObserver(observer))
638	-	stdio := sshstdio.NewTransport(os.Stdin, os.Stdout, nil)
639		return proxyTransports(ctx, stdio, transport, log)	748		return proxyTransports(ctx, stdio, transport, log)
640		}	749		}
641			750

@@ -727,6 +836,13 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) error {
727		defer func() { _ = logCloser.Close() }()	836		defer func() { _ = logCloser.Close() }()
728		log.Debug("udp proxy starting", "session", session)	837		log.Debug("udp proxy starting", "session", session)
729			838
			839	+	observer, observerCloser, err := performanceTrace(clock.New())
			840	+	if err != nil {
			841	+	return fmt.Errorf("vev: performance trace: %w", err)
			842	+	}
			843	+	if observerCloser != nil {
			844	+	defer func() { _ = observerCloser.Close() }()
			845	+	}
730		portRange, err := parseUDPPortRange(os.Getenv(envUDPPortRange))	846		portRange, err := parseUDPPortRange(os.Getenv(envUDPPortRange))
731		if err != nil {	847		if err != nil {
732		return err	848		return err

@@ -740,14 +856,16 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) error {
740		if _, err := rand.Read(key); err != nil {	856		if _, err := rand.Read(key); err != nil {
741		return err	857		return err
742		}	858		}
743	-	daemonTr, err := ensureDaemon(ctx, ipc.SocketDir(), realDial, realSpawn, defaultBackoff)	859	+	daemonTr, err := ensureDaemon(ctx, ipc.SocketDir(), func(ctx context.Context, dir string) (ports.Transport, error) {
			860	+	return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(observer))
			861	+	}, realSpawn, defaultBackoff)
744		if err != nil {	862		if err != nil {
745		return err	863		return err
746		}	864		}
747		defer func() { _ = daemonTr.Close() }()	865		defer func() { _ = daemonTr.Close() }()
748		udpOptions := udpProxyClientTransportOptions	866		udpOptions := udpProxyClientTransportOptions
749		udpOptions.Observe = dgram.DiagnosticLogObserver(log)	867		udpOptions.Observe = dgram.DiagnosticLogObserver(log)
750	-	dg, err := dgram.NewTransportWithOptions(conn, nil, key, 2, 1, udpOptions)	868	+	dg, err := dgram.NewTransportWithOptions(conn, nil, key, 2, 1, udpOptions, dgram.WithRuntimeObserver(observer))
751		if err != nil {	869		if err != nil {
752		return err	870		return err
753		}	871		}

@@ -890,8 +1008,8 @@ func proxyTransports(ctx context.Context, a, b ports.Transport, log *slog.Logger
890			1008
891		// runList prints the daemon's session listing. With no daemon running, it	1009		// runList prints the daemon's session listing. With no daemon running, it
892		// falls back to the persisted stopped-session records.	1010		// falls back to the persisted stopped-session records.
893	-	func runList(_ context.Context) error {	1011	+	func runList(ctx context.Context) error {
894	-	transport, err := realDial(ipc.SocketDir())	1012	+	transport, err := realDial(ctx, ipc.SocketDir())
895		if err != nil {	1013		if err != nil {
896		records, loadErr := persist.LoadReadOnly(platform.StateDir())	1014		records, loadErr := persist.LoadReadOnly(platform.StateDir())
897		if loadErr != nil {	1015		if loadErr != nil {

@@ -955,8 +1073,8 @@ func printSessions(w io.Writer, sessions []ports.SessionInfo) {
955		}	1073		}
956			1074
957		// runKill asks the daemon to terminate a named session, every session, or the daemon.	1075		// runKill asks the daemon to terminate a named session, every session, or the daemon.
958	-	func runKill(_ context.Context, name string, all, daemon bool) error {	1076	+	func runKill(ctx context.Context, name string, all, daemon bool) error {
959	-	transport, err := realDial(ipc.SocketDir())	1077	+	transport, err := realDial(ctx, ipc.SocketDir())
960		if err != nil {	1078		if err != nil {
961		if name != "" && !all && !daemon {	1079		if name != "" && !all && !daemon {
962		storePath := persist.StorePath(platform.StateDir())	1080		storePath := persist.StorePath(platform.StateDir())

M internal/app/spawn.go +14 -5

 // dialFunc dials the daemon socket in dir. Injected so tests can drive the
 // spawn/backoff logic without a real socket.
-type dialFunc func(dir string) (ports.Transport, error)
+type dialFunc func(ctx context.Context, dir string) (ports.Transport, error)
 // spawnFunc launches a detached daemon process. Injected so tests never
 // re-exec a real binary.
 // spawner via an mkdir lock (losers simply wait), then retry-dials with
 // backoff until the socket is live or the budget expires.
 func ensureDaemon(ctx context.Context, dir string, dial dialFunc, spawn spawnFunc, cfg backoffConfig) (ports.Transport, error) {
-	if t, err := dial(dir); err == nil {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	if t, err := dial(ctx, dir); err == nil {
 		slog.Debug("daemon already reachable", "socket_dir", dir)
 		return t, nil
 	}
 		// rather than spawning a second daemon.
 		defer release()
 		slog.Info("spawning daemon", "socket_dir", dir)
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
 		if err := spawn(); err != nil {
 			slog.Error("daemon spawn failed", "err", err)
 			return nil, fmt.Errorf("vev: spawning daemon: %w", err)
 	deadline := time.Now().Add(cfg.total)
 	backoff := cfg.initial
 	for {
-		if t, err := dial(dir); err == nil {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		if t, err := dial(ctx, dir); err == nil {
 			return t, nil
 		}
 		if !time.Now().Before(deadline) {
 }
 // realDial is the production dialer.
-func realDial(dir string) (ports.Transport, error) {
+func realDial(ctx context.Context, dir string) (ports.Transport, error) {
-	return ipc.Dial(dir)
+	return ipc.DialContext(ctx, dir)
 }
 // realSpawn re-execs this binary as a detached daemon: a new session

M internal/app/spawn_test.go +53 -6


@@ -91,7 +91,7 @@ func TestAcquireSpawnLockTakesOverStaleLock(t *testing.T) {
91		func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {	91		func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {
92		want := portsmocks.NewMockTransport(t)	92		want := portsmocks.NewMockTransport(t)
93		var calls atomic.Int32	93		var calls atomic.Int32
94	-	dial := func(string) (ports.Transport, error) {	94	+	dial := func(context.Context, string) (ports.Transport, error) {
95		if calls.Add(1) <= 3 {	95		if calls.Add(1) <= 3 {
96		return nil, errDialFailed	96		return nil, errDialFailed
97		}	97		}

@@ -111,7 +111,7 @@ func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {
111		}	111		}
112			112
113		func TestRetryDialPermanentFailure(t *testing.T) {	113		func TestRetryDialPermanentFailure(t *testing.T) {
114	-	dial := func(string) (ports.Transport, error) { return nil, errDialFailed }	114	+	dial := func(context.Context, string) (ports.Transport, error) { return nil, errDialFailed }
115		_, err := retryDial(context.Background(), t.TempDir(), dial, fastBackoff)	115		_, err := retryDial(context.Background(), t.TempDir(), dial, fastBackoff)
116		if !errors.Is(err, ErrDaemonUnreachable) {	116		if !errors.Is(err, ErrDaemonUnreachable) {
117		t.Fatalf("retryDial error = %v, want ErrDaemonUnreachable", err)	117		t.Fatalf("retryDial error = %v, want ErrDaemonUnreachable", err)

@@ -121,18 +121,65 @@ func TestRetryDialPermanentFailure(t *testing.T) {
121		func TestRetryDialContextCancelled(t *testing.T) {	121		func TestRetryDialContextCancelled(t *testing.T) {
122		ctx, cancel := context.WithCancel(context.Background())	122		ctx, cancel := context.WithCancel(context.Background())
123		cancel()	123		cancel()
124	-	dial := func(string) (ports.Transport, error) { return nil, errDialFailed }	124	+	var calls atomic.Int32
			125	+	dial := func(context.Context, string) (ports.Transport, error) {
			126	+	calls.Add(1)
			127	+	return nil, errDialFailed
			128	+	}
125		_, err := retryDial(ctx, t.TempDir(), dial, fastBackoff)	129		_, err := retryDial(ctx, t.TempDir(), dial, fastBackoff)
126		if !errors.Is(err, context.Canceled) {	130		if !errors.Is(err, context.Canceled) {
127		t.Fatalf("retryDial error = %v, want context.Canceled", err)	131		t.Fatalf("retryDial error = %v, want context.Canceled", err)
128		}	132		}
			133	+	if calls.Load() != 0 {
			134	+	t.Fatalf("dial called %d times for canceled context, want 0", calls.Load())
			135	+	}
			136	+	}
			137	+
			138	+	func TestEnsureDaemonContextCancelledDoesNotDialOrSpawn(t *testing.T) {
			139	+	ctx, cancel := context.WithCancel(context.Background())
			140	+	cancel()
			141	+	dial := func(context.Context, string) (ports.Transport, error) {
			142	+	t.Fatal("ensureDaemon dialed with a canceled context")
			143	+	return nil, nil
			144	+	}
			145	+	spawn := func() error {
			146	+	t.Fatal("ensureDaemon spawned with a canceled context")
			147	+	return nil
			148	+	}
			149	+
			150	+	_, err := ensureDaemon(ctx, t.TempDir(), dial, spawn, fastBackoff)
			151	+	if !errors.Is(err, context.Canceled) {
			152	+	t.Fatalf("ensureDaemon error = %v, want context.Canceled", err)
			153	+	}
			154	+	}
			155	+
			156	+	func TestEnsureDaemonDoesNotSpawnWhenContextCancelsAfterFailedDial(t *testing.T) {
			157	+	ctx, cancel := context.WithCancel(context.Background())
			158	+	defer cancel()
			159	+	var spawnCalls atomic.Int32
			160	+	dial := func(context.Context, string) (ports.Transport, error) {
			161	+	cancel()
			162	+	return nil, errDialFailed
			163	+	}
			164	+	spawn := func() error {
			165	+	spawnCalls.Add(1)
			166	+	return nil
			167	+	}
			168	+
			169	+	_, err := ensureDaemon(ctx, filepath.Join(t.TempDir(), "vev"), dial, spawn, fastBackoff)
			170	+	if !errors.Is(err, context.Canceled) {
			171	+	t.Fatalf("ensureDaemon error = %v, want context.Canceled", err)
			172	+	}
			173	+	if got := spawnCalls.Load(); got != 0 {
			174	+	t.Fatalf("spawn called %d times after canceled failed dial, want 0", got)
			175	+	}
129		}	176		}
130			177
131		func TestEnsureDaemonSpawnsThenDials(t *testing.T) {	178		func TestEnsureDaemonSpawnsThenDials(t *testing.T) {
132		dir := filepath.Join(t.TempDir(), "vev")	179		dir := filepath.Join(t.TempDir(), "vev")
133		want := portsmocks.NewMockTransport(t)	180		want := portsmocks.NewMockTransport(t)
134		var dialCalls, spawnCalls atomic.Int32	181		var dialCalls, spawnCalls atomic.Int32
135	-	dial := func(string) (ports.Transport, error) {	182	+	dial := func(context.Context, string) (ports.Transport, error) {
136		// First (pre-spawn) dial fails; the post-spawn retry succeeds.	183		// First (pre-spawn) dial fails; the post-spawn retry succeeds.
137		if dialCalls.Add(1) == 1 {	184		if dialCalls.Add(1) == 1 {
138		return nil, errDialFailed	185		return nil, errDialFailed

@@ -163,7 +210,7 @@ func TestEnsureDaemonSpawnsThenDials(t *testing.T) {
163		func TestEnsureDaemonReturnsExistingDaemon(t *testing.T) {	210		func TestEnsureDaemonReturnsExistingDaemon(t *testing.T) {
164		want := portsmocks.NewMockTransport(t)	211		want := portsmocks.NewMockTransport(t)
165		var spawnCalls atomic.Int32	212		var spawnCalls atomic.Int32
166	-	dial := func(string) (ports.Transport, error) { return want, nil }	213	+	dial := func(context.Context, string) (ports.Transport, error) { return want, nil }
167		spawn := func() error { spawnCalls.Add(1); return nil }	214		spawn := func() error { spawnCalls.Add(1); return nil }
168			215
169		got, err := ensureDaemon(context.Background(), t.TempDir(), dial, spawn, fastBackoff)	216		got, err := ensureDaemon(context.Background(), t.TempDir(), dial, spawn, fastBackoff)

@@ -180,7 +227,7 @@ func TestEnsureDaemonReturnsExistingDaemon(t *testing.T) {
180			227
181		func TestEnsureDaemonSpawnFailure(t *testing.T) {	228		func TestEnsureDaemonSpawnFailure(t *testing.T) {
182		spawnErr := errors.New("exec boom")	229		spawnErr := errors.New("exec boom")
183	-	dial := func(string) (ports.Transport, error) { return nil, errDialFailed }	230	+	dial := func(context.Context, string) (ports.Transport, error) { return nil, errDialFailed }
184		spawn := func() error { return spawnErr }	231		spawn := func() error { return spawnErr }
185			232
186		_, err := ensureDaemon(context.Background(), filepath.Join(t.TempDir(), "vev"), dial, spawn, fastBackoff)	233		_, err := ensureDaemon(context.Background(), filepath.Join(t.TempDir(), "vev"), dial, spawn, fastBackoff)

A internal/app/transport_observability_test.go +82 -0

@@ -0,0 +1,82 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/json"
	6	+	"os"
	7	+	"path/filepath"
	8	+	"strings"
	9	+	"testing"
	10	+	"time"
	11	+
	12	+	"github.com/bnema/vev/internal/ports"
	13	+	portsmocks "github.com/bnema/vev/internal/ports/mocks"
	14	+	)
	15	+
	16	+	func TestPerformanceTraceUsesHarnessProcessMapping(t *testing.T) {
	17	+	path := filepath.Join(t.TempDir(), "process.jsonl")
	18	+	t.Setenv("VEV_PERF_TRACE", path)
	19	+	t.Setenv("VEV_PERF_PROCESS_ID", "idle-local-r001-daemon-01")
	20	+	t.Setenv("VEV_PERF_SCENARIO", "1x4-idle-local")
	21	+	t.Setenv("VEV_PERF_RUN", "1")
	22	+
	23	+	clock := portsmocks.NewMockClock(t)
	24	+	clock.EXPECT().Now().Return(time.Unix(0, 1)).Once()
	25	+	observer, closer, err := performanceTrace(clock)
	26	+	if err != nil {
	27	+	t.Fatal(err)
	28	+	}
	29	+	if observer == nil \|\| closer == nil {
	30	+	t.Fatal("performance trace was not configured")
	31	+	}
	32	+	if _, ok := observer.(ports.SerializedRuntimeObserver); !ok {
	33	+	t.Fatalf("performance trace observer = %T, want serialized owner", observer)
	34	+	}
	35	+	observer.ObserveRuntime(ports.RuntimeMark{Schema: ports.RuntimeMarkSchema, Component: "daemon", Scenario: "runtime", Run: 1, Kind: ports.RuntimeEmitEnd, Valid: true})
	36	+	if err := closer.Close(); err != nil {
	37	+	t.Fatal(err)
	38	+	}
	39	+	if err := closer.Close(); err != nil {
	40	+	t.Fatal(err)
	41	+	}
	42	+	f, err := os.Open(path)
	43	+	if err != nil {
	44	+	t.Fatal(err)
	45	+	}
	46	+	defer f.Close()
	47	+	var mark struct {
	48	+	ProcessID string `json:"process_id"`
	49	+	Scenario string `json:"scenario"`
	50	+	Run uint64 `json:"run"`
	51	+	Sequence uint64 `json:"sequence"`
	52	+	RequestID uint64 `json:"request_id"`
	53	+	Epoch uint64 `json:"epoch"`
	54	+	}
	55	+	if err := json.NewDecoder(bufio.NewReader(f)).Decode(&mark); err != nil {
	56	+	t.Fatal(err)
	57	+	}
	58	+	if mark.ProcessID != "idle-local-r001-daemon-01" \|\| mark.Scenario != "1x4-idle-local" \|\| mark.Run != 1 \|\| mark.Sequence == 0 \|\| mark.RequestID == 0 \|\| mark.Epoch == 0 {
	59	+	t.Fatalf("mark does not match harness mapping: %+v", mark)
	60	+	}
	61	+	}
	62	+
	63	+	func TestPerformanceTraceAppWiresOneObserverPerProcess(t *testing.T) {
	64	+	source, err := os.ReadFile("run.go")
	65	+	if err != nil {
	66	+	t.Fatalf("ReadFile(run.go): %v", err)
	67	+	}
	68	+	text := string(source)
	69	+	for _, required := range []string{
	70	+	"VEV_PERF_TRACE",
	71	+	"VEV_PERF_PROCESS_ID",
	72	+	"observability.NewJSONL",
	73	+	"WithRuntimeObserver",
	74	+	} {
	75	+	if !strings.Contains(text, required) {
	76	+	t.Errorf("runtime trace wiring missing %q", required)
	77	+	}
	78	+	}
	79	+	if strings.Contains(text, "WithRuntimeObserver(clock") \|\| strings.Contains(text, "WithRuntimeObserver(clk") {
	80	+	t.Error("consumer observer configuration must not accept a clock")
	81	+	}
	82	+	}

M internal/ports/mocks/mocks.go +67 -0


@@ -3129,3 +3129,70 @@ func (_c *MockRemoteDialerFactory_DialerForRemote_Call) RunAndReturn(run func(ta
3129	_c.Call.Return(run)	3129		_c.Call.Return(run)
3130	return _c	3130		return _c
3131	}	3131		}
		3132	+
		3133	+	// NewMockRuntimeObserver creates a new instance of MockRuntimeObserver. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
		3134	+	// The first argument is typically a *testing.T value.
		3135	+	func NewMockRuntimeObserver(t interface {
		3136	+	mock.TestingT
		3137	+	Cleanup(func())
		3138	+	}) *MockRuntimeObserver {
		3139	+	mock := &MockRuntimeObserver{}
		3140	+	mock.Mock.Test(t)
		3141	+
		3142	+	t.Cleanup(func() { mock.AssertExpectations(t) })
		3143	+
		3144	+	return mock
		3145	+	}
		3146	+
		3147	+	// MockRuntimeObserver is an autogenerated mock type for the RuntimeObserver type
		3148	+	type MockRuntimeObserver struct {
		3149	+	mock.Mock
		3150	+	}
		3151	+
		3152	+	type MockRuntimeObserver_Expecter struct {
		3153	+	mock *mock.Mock
		3154	+	}
		3155	+
		3156	+	func (_m MockRuntimeObserver) EXPECT() MockRuntimeObserver_Expecter {
		3157	+	return &MockRuntimeObserver_Expecter{mock: &_m.Mock}
		3158	+	}
		3159	+
		3160	+	// ObserveRuntime provides a mock function for the type MockRuntimeObserver
		3161	+	func (_mock *MockRuntimeObserver) ObserveRuntime(runtimeMark ports.RuntimeMark) {
		3162	+	_mock.Called(runtimeMark)
		3163	+	return
		3164	+	}
		3165	+
		3166	+	// MockRuntimeObserver_ObserveRuntime_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ObserveRuntime'
		3167	+	type MockRuntimeObserver_ObserveRuntime_Call struct {
		3168	+	*mock.Call
		3169	+	}
		3170	+
		3171	+	// ObserveRuntime is a helper method to define mock.On call
		3172	+	// - runtimeMark ports.RuntimeMark
		3173	+	func (_e MockRuntimeObserver_Expecter) ObserveRuntime(runtimeMark any) MockRuntimeObserver_ObserveRuntime_Call {
		3174	+	return &MockRuntimeObserver_ObserveRuntime_Call{Call: _e.mock.On("ObserveRuntime", runtimeMark)}
		3175	+	}
		3176	+
		3177	+	func (_c MockRuntimeObserver_ObserveRuntime_Call) Run(run func(runtimeMark ports.RuntimeMark)) MockRuntimeObserver_ObserveRuntime_Call {
		3178	+	_c.Call.Run(func(args mock.Arguments) {
		3179	+	var arg0 ports.RuntimeMark
		3180	+	if args[0] != nil {
		3181	+	arg0 = args[0].(ports.RuntimeMark)
		3182	+	}
		3183	+	run(
		3184	+	arg0,
		3185	+	)
		3186	+	})
		3187	+	return _c
		3188	+	}
		3189	+
		3190	+	func (_c MockRuntimeObserver_ObserveRuntime_Call) Return() MockRuntimeObserver_ObserveRuntime_Call {
		3191	+	_c.Call.Return()
		3192	+	return _c
		3193	+	}
		3194	+
		3195	+	func (_c MockRuntimeObserver_ObserveRuntime_Call) RunAndReturn(run func(runtimeMark ports.RuntimeMark)) MockRuntimeObserver_ObserveRuntime_Call {
		3196	+	_c.Run(run)
		3197	+	return _c
		3198	+	}

A internal/ports/runtime_observer.go +297 -0

@@ -0,0 +1,297 @@
	1	+	package ports
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"sync"
	6	+	"sync/atomic"
	7	+	)
	8	+
	9	+	// RuntimeMarkSchema is the version of the process-local performance trace.
	10	+	const RuntimeMarkSchema uint16 = 1
	11	+
	12	+	// RuntimeMarkKind identifies a process-local runtime boundary. It deliberately
	13	+	// contains no protocol payload, cells, or rendering policy.
	14	+	type RuntimeMarkKind string
	15	+
	16	+	const (
	17	+	RuntimeInputInjected RuntimeMarkKind = "input_injected"
	18	+	RuntimeResizeRequested RuntimeMarkKind = "resize_requested"
	19	+	RuntimeResizeCommitted RuntimeMarkKind = "resize_committed"
	20	+	RuntimeCaptureStart RuntimeMarkKind = "capture_start"
	21	+	RuntimeCaptureEnd RuntimeMarkKind = "capture_end"
	22	+	RuntimeComposeStart RuntimeMarkKind = "compose_start"
	23	+	RuntimeComposeEnd RuntimeMarkKind = "compose_end"
	24	+	RuntimeDiffStart RuntimeMarkKind = "diff_start"
	25	+	RuntimeDiffEnd RuntimeMarkKind = "diff_end"
	26	+	RuntimeQueueEnqueued RuntimeMarkKind = "queue_enqueued"
	27	+	RuntimeQueueDequeued RuntimeMarkKind = "queue_dequeued"
	28	+	RuntimeACKBlockedStart RuntimeMarkKind = "ack_blocked_start"
	29	+	RuntimeACKBlockedEnd RuntimeMarkKind = "ack_blocked_end"
	30	+	RuntimeEmitStart RuntimeMarkKind = "emit_start"
	31	+	RuntimeEmitEnd RuntimeMarkKind = "emit_end"
	32	+	RuntimeAdapterSendStart RuntimeMarkKind = "adapter_send_start"
	33	+	RuntimeAdapterSendEnd RuntimeMarkKind = "adapter_send_end"
	34	+	RuntimeAdapterReceiveStart RuntimeMarkKind = "adapter_receive_start"
	35	+	RuntimeAdapterReceiveEnd RuntimeMarkKind = "adapter_receive_end"
	36	+	RuntimeTerminalFlushed RuntimeMarkKind = "terminal_flushed"
	37	+	RuntimeTransportDiagnostic RuntimeMarkKind = "transport_diagnostic"
	38	+	)
	39	+
	40	+	// RuntimeMark is a process-local observation record. Tick is intentionally
	41	+	// left zero by producers: the concrete process observer owns timestamping.
	42	+	type RuntimeMark struct {
	43	+	Schema uint16
	44	+	ProcessID string
	45	+	Component string
	46	+	Scenario string
	47	+	Run uint64
	48	+	Sequence uint64
	49	+	RequestID uint64
	50	+	Epoch uint64
	51	+	Kind RuntimeMarkKind
	52	+	Tick int64
	53	+	Bytes uint64
	54	+	Fragments uint64
	55	+	Retransmits uint64
	56	+	Pending uint64
	57	+	AckRTTNanos int64
	58	+	Valid bool
	59	+	}
	60	+
	61	+	// RuntimeObserver receives process-local marks. Implementations must not let
	62	+	// a producer provide a timestamp in another clock domain.
	63	+	type RuntimeObserver interface{ ObserveRuntime(RuntimeMark) }
	64	+
	65	+	// SerializedRuntimeObserver accepts marks without waiting for observer I/O.
	66	+	// Flush and Close wait until every accepted mark has reached Observer. Close
	67	+	// is idempotent and rejects later marks.
	68	+	type SerializedRuntimeObserver interface {
	69	+	RuntimeObserver
	70	+	Flush()
	71	+	Close()
	72	+	}
	73	+
	74	+	// NewSerializedRuntimeObserver isolates producers from a potentially slow
	75	+	// observer with one ordered worker. capacity bounds queued marks. If the queue
	76	+	// fills, dropped marks are represented by one ordered diagnostic mark before
	77	+	// reporting resumes, so a trace can never silently claim completeness.
	78	+	func NewSerializedRuntimeObserver(observer RuntimeObserver, capacity int) SerializedRuntimeObserver {
	79	+	if observer == nil {
	80	+	return nil
	81	+	}
	82	+	if capacity < 1 {
	83	+	capacity = 1
	84	+	}
	85	+	o := &serializedRuntimeObserver{observer: observer, capacity: capacity, done: make(chan struct{})}
	86	+	o.cond = sync.NewCond(&o.mu)
	87	+	go o.run()
	88	+	return o
	89	+	}
	90	+
	91	+	// EnsureSerializedRuntimeObserver returns observer's existing serialized
	92	+	// owner when it has one. Otherwise it creates an owned worker. Callers close
	93	+	// the result only when owned is true, preventing nested workers and duplicate
	94	+	// lifecycle ownership when application wiring already serialized the process
	95	+	// observer.
	96	+	func EnsureSerializedRuntimeObserver(observer RuntimeObserver, capacity int) (serialized SerializedRuntimeObserver, owned bool) {
	97	+	if observer == nil {
	98	+	return nil, false
	99	+	}
	100	+	if serialized, ok := observer.(SerializedRuntimeObserver); ok {
	101	+	return serialized, false
	102	+	}
	103	+	return NewSerializedRuntimeObserver(observer, capacity), true
	104	+	}
	105	+
	106	+	type serializedRuntimeObserver struct {
	107	+	observer RuntimeObserver
	108	+	capacity int
	109	+
	110	+	mu sync.Mutex
	111	+	cond *sync.Cond
	112	+	marks []RuntimeMark
	113	+	dropped uint64
	114	+	closed bool
	115	+	active bool
	116	+	done chan struct{}
	117	+	}
	118	+
	119	+	func (o *serializedRuntimeObserver) ObserveRuntime(mark RuntimeMark) {
	120	+	if o == nil {
	121	+	return
	122	+	}
	123	+	o.mu.Lock()
	124	+	defer o.mu.Unlock()
	125	+	if o.closed {
	126	+	return
	127	+	}
	128	+	if o.dropped != 0 {
	129	+	// Reserve space for both the gap marker and the first post-gap mark.
	130	+	// This preserves the producer order around a bounded-queue loss.
	131	+	if len(o.marks)+2 > o.capacity {
	132	+	o.dropped++
	133	+	return
	134	+	}
	135	+	o.marks = append(o.marks, runtimeObserverGapMark(o.dropped), mark)
	136	+	o.dropped = 0
	137	+	o.cond.Signal()
	138	+	return
	139	+	}
	140	+	if len(o.marks) == o.capacity {
	141	+	o.dropped = 1
	142	+	return
	143	+	}
	144	+	o.marks = append(o.marks, mark)
	145	+	o.cond.Signal()
	146	+	}
	147	+
	148	+	func runtimeObserverGapMark(dropped uint64) RuntimeMark {
	149	+	return NewRuntimeMark("observer", RuntimeTransportDiagnostic, dropped, false)
	150	+	}
	151	+
	152	+	func (o *serializedRuntimeObserver) Flush() {
	153	+	if o == nil {
	154	+	return
	155	+	}
	156	+	o.mu.Lock()
	157	+	for len(o.marks) != 0 \|\| o.active \|\| o.dropped != 0 {
	158	+	o.cond.Signal()
	159	+	o.cond.Wait()
	160	+	}
	161	+	closed := o.closed
	162	+	o.mu.Unlock()
	163	+	if closed {
	164	+	<-o.done
	165	+	}
	166	+	}
	167	+
	168	+	func (o *serializedRuntimeObserver) Close() {
	169	+	if o == nil {
	170	+	return
	171	+	}
	172	+	o.mu.Lock()
	173	+	if !o.closed {
	174	+	o.closed = true
	175	+	o.cond.Broadcast()
	176	+	}
	177	+	o.mu.Unlock()
	178	+	<-o.done
	179	+	}
	180	+
	181	+	func (o *serializedRuntimeObserver) run() {
	182	+	defer close(o.done)
	183	+	for {
	184	+	o.mu.Lock()
	185	+	for len(o.marks) == 0 && o.dropped == 0 && !o.closed {
	186	+	o.cond.Wait()
	187	+	}
	188	+	if len(o.marks) == 0 && o.dropped != 0 {
	189	+	o.marks = append(o.marks, runtimeObserverGapMark(o.dropped))
	190	+	o.dropped = 0
	191	+	}
	192	+	if len(o.marks) == 0 && o.closed {
	193	+	o.cond.Broadcast()
	194	+	o.mu.Unlock()
	195	+	return
	196	+	}
	197	+	mark := o.marks[0]
	198	+	o.marks = o.marks[1:]
	199	+	o.active = true
	200	+	o.mu.Unlock()
	201	+
	202	+	o.observer.ObserveRuntime(mark)
	203	+
	204	+	o.mu.Lock()
	205	+	o.active = false
	206	+	o.cond.Broadcast()
	207	+	o.mu.Unlock()
	208	+	}
	209	+	}
	210	+
	211	+	// RuntimeMarkValid reports whether a mark has all correlation fields required
	212	+	// by the JSONL schema. Tick is checked separately by the timestamp owner.
	213	+	func RuntimeMarkValid(mark RuntimeMark) bool {
	214	+	if mark.Schema != RuntimeMarkSchema \|\| mark.Component == "" \|\| mark.Scenario == "" \|\| mark.Run == 0 \|\| mark.Sequence == 0 \|\| mark.RequestID == 0 \|\| mark.Epoch == 0 {
	215	+	return false
	216	+	}
	217	+	switch mark.Kind {
	218	+	case RuntimeInputInjected, RuntimeResizeRequested, RuntimeResizeCommitted,
	219	+	RuntimeCaptureStart, RuntimeCaptureEnd, RuntimeComposeStart, RuntimeComposeEnd,
	220	+	RuntimeDiffStart, RuntimeDiffEnd, RuntimeQueueEnqueued, RuntimeQueueDequeued,
	221	+	RuntimeACKBlockedStart, RuntimeACKBlockedEnd, RuntimeEmitStart, RuntimeEmitEnd,
	222	+	RuntimeAdapterSendStart, RuntimeAdapterSendEnd, RuntimeAdapterReceiveStart,
	223	+	RuntimeAdapterReceiveEnd, RuntimeTerminalFlushed, RuntimeTransportDiagnostic:
	224	+	return true
	225	+	default:
	226	+	return false
	227	+	}
	228	+	}
	229	+
	230	+	// RuntimeCorrelation identifies one attempted operation. It is deliberately
	231	+	// copied to both endpoints of a span; process ID remains owned by the sink.
	232	+	type RuntimeCorrelation struct {
	233	+	Scenario string
	234	+	Run, Sequence, RequestID, Epoch uint64
	235	+	}
	236	+
	237	+	// RuntimeCorrelationInputs are supplied once by process orchestration. They
	238	+	// are deliberately data-only: consumer APIs continue to receive only their
	239	+	// one RuntimeObserver argument and never a clock or tick.
	240	+	type RuntimeCorrelationInputs struct {
	241	+	Scenario string
	242	+	Run uint64
	243	+	}
	244	+
	245	+	// NewRuntimeCorrelationObserver binds every mark from one OS process to its
	246	+	// harness manifest scenario/run. Operation IDs remain producer-owned when
	247	+	// supplied; missing IDs are assigned from the process-wide allocator.
	248	+	func NewRuntimeCorrelationObserver(observer RuntimeObserver, inputs RuntimeCorrelationInputs) (RuntimeObserver, error) {
	249	+	if observer == nil \|\| inputs.Scenario == "" \|\| inputs.Run == 0 {
	250	+	return nil, errors.New("invalid runtime correlation inputs")
	251	+	}
	252	+	return &runtimeCorrelationObserver{observer: observer, inputs: inputs}, nil
	253	+	}
	254	+
	255	+	type runtimeCorrelationObserver struct {
	256	+	observer RuntimeObserver
	257	+	inputs RuntimeCorrelationInputs
	258	+	}
	259	+
	260	+	func (o *runtimeCorrelationObserver) ObserveRuntime(mark RuntimeMark) {
	261	+	if o == nil \|\| o.observer == nil {
	262	+	return
	263	+	}
	264	+	mark.Scenario, mark.Run = o.inputs.Scenario, o.inputs.Run
	265	+	if mark.Sequence == 0 {
	266	+	mark.Sequence = runtimeMarkSequence.Add(1)
	267	+	}
	268	+	if mark.RequestID == 0 {
	269	+	mark.RequestID = mark.Sequence
	270	+	}
	271	+	if mark.Epoch == 0 {
	272	+	mark.Epoch = mark.RequestID
	273	+	}
	274	+	o.observer.ObserveRuntime(mark)
	275	+	}
	276	+
	277	+	var runtimeMarkSequence atomic.Uint64
	278	+
	279	+	// NewRuntimeCorrelation makes a unique, process-local operation identity.
	280	+	// Components with a domain request/epoch should use those values directly via
	281	+	// NewRuntimeMarkWithCorrelation instead of inventing a second identity.
	282	+	func NewRuntimeCorrelation() RuntimeCorrelation {
	283	+	sequence := runtimeMarkSequence.Add(1)
	284	+	return RuntimeCorrelation{Scenario: "runtime", Run: 1, Sequence: sequence, RequestID: sequence, Epoch: sequence}
	285	+	}
	286	+
	287	+	// NewRuntimeMarkWithCorrelation returns a timestamp-free mark carrying the
	288	+	// supplied operation identity. Start/end callers must reuse correlation.
	289	+	func NewRuntimeMarkWithCorrelation(component string, correlation RuntimeCorrelation, kind RuntimeMarkKind, bytes uint64, valid bool) RuntimeMark {
	290	+	return RuntimeMark{Schema: RuntimeMarkSchema, Component: component, Scenario: correlation.Scenario, Run: correlation.Run, Sequence: correlation.Sequence, RequestID: correlation.RequestID, Epoch: correlation.Epoch, Kind: kind, Bytes: bytes, Valid: valid}
	291	+	}
	292	+
	293	+	// NewRuntimeMark is retained for one-shot marks. Unlike the old helper it
	294	+	// never returns constant correlation IDs.
	295	+	func NewRuntimeMark(component string, kind RuntimeMarkKind, bytes uint64, valid bool) RuntimeMark {
	296	+	return NewRuntimeMarkWithCorrelation(component, NewRuntimeCorrelation(), kind, bytes, valid)
	297	+	}

A internal/ports/runtime_observer_test.go +204 -0

@@ -0,0 +1,204 @@
	1	+	package ports
	2	+
	3	+	import (
	4	+	"reflect"
	5	+	"sync"
	6	+	"testing"
	7	+	"time"
	8	+	)
	9	+
	10	+	func TestRuntimeObserverContract(t *testing.T) {
	11	+	wantFields := []struct {
	12	+	name string
	13	+	typ reflect.Type
	14	+	}{
	15	+	{"Schema", reflect.TypeFor[uint16]()},
	16	+	{"ProcessID", reflect.TypeFor[string]()},
	17	+	{"Component", reflect.TypeFor[string]()},
	18	+	{"Scenario", reflect.TypeFor[string]()},
	19	+	{"Run", reflect.TypeFor[uint64]()},
	20	+	{"Sequence", reflect.TypeFor[uint64]()},
	21	+	{"RequestID", reflect.TypeFor[uint64]()},
	22	+	{"Epoch", reflect.TypeFor[uint64]()},
	23	+	{"Kind", reflect.TypeFor[RuntimeMarkKind]()},
	24	+	{"Tick", reflect.TypeFor[int64]()},
	25	+	{"Bytes", reflect.TypeFor[uint64]()},
	26	+	{"Fragments", reflect.TypeFor[uint64]()},
	27	+	{"Retransmits", reflect.TypeFor[uint64]()},
	28	+	{"Pending", reflect.TypeFor[uint64]()},
	29	+	{"AckRTTNanos", reflect.TypeFor[int64]()},
	30	+	{"Valid", reflect.TypeFor[bool]()},
	31	+	}
	32	+
	33	+	typ := reflect.TypeFor[RuntimeMark]()
	34	+	if typ.NumField() != len(wantFields) {
	35	+	t.Fatalf("RuntimeMark has %d fields, want exactly %d", typ.NumField(), len(wantFields))
	36	+	}
	37	+	for i, want := range wantFields {
	38	+	field := typ.Field(i)
	39	+	if field.Name != want.name \|\| field.Type != want.typ {
	40	+	t.Fatalf("RuntimeMark field %d = %s %v, want %s %v", i, field.Name, field.Type, want.name, want.typ)
	41	+	}
	42	+	}
	43	+
	44	+	var _ RuntimeObserver = runtimeObserverFunc(func(RuntimeMark) {})
	45	+	}
	46	+
	47	+	func TestSerializedRuntimeObserverFlushesInOrderAndCloses(t *testing.T) {
	48	+	var mu sync.Mutex
	49	+	var got []RuntimeMark
	50	+	observer := NewSerializedRuntimeObserver(runtimeObserverFunc(func(mark RuntimeMark) {
	51	+	mu.Lock()
	52	+	got = append(got, mark)
	53	+	mu.Unlock()
	54	+	}), 2)
	55	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeEmitStart, 1, true))
	56	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeEmitEnd, 1, true))
	57	+	observer.Flush()
	58	+	observer.Close()
	59	+	observer.Close()
	60	+
	61	+	mu.Lock()
	62	+	defer mu.Unlock()
	63	+	if len(got) != 2 \|\| got[0].Kind != RuntimeEmitStart \|\| got[1].Kind != RuntimeEmitEnd {
	64	+	t.Fatalf("serialized marks = %#v", got)
	65	+	}
	66	+	}
	67	+
	68	+	func TestEnsureSerializedRuntimeObserverReusesExistingOwner(t *testing.T) {
	69	+	existing := NewSerializedRuntimeObserver(runtimeObserverFunc(func(RuntimeMark) {}), 1)
	70	+	reporter, owned := EnsureSerializedRuntimeObserver(existing, 1)
	71	+	if owned \|\| reporter != existing {
	72	+	t.Fatalf("EnsureSerializedRuntimeObserver() = (%T, %t), want existing unowned reporter", reporter, owned)
	73	+	}
	74	+	existing.Close()
	75	+	}
	76	+
	77	+	func TestSerializedRuntimeObserverReportsBoundedQueueLoss(t *testing.T) {
	78	+	entered := make(chan struct{})
	79	+	release := make(chan struct{})
	80	+	var mu sync.Mutex
	81	+	var got []RuntimeMark
	82	+	observer := NewSerializedRuntimeObserver(runtimeObserverFunc(func(mark RuntimeMark) {
	83	+	if mark.Kind == RuntimeEmitStart {
	84	+	close(entered)
	85	+	<-release
	86	+	}
	87	+	mu.Lock()
	88	+	got = append(got, mark)
	89	+	mu.Unlock()
	90	+	}), 2)
	91	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeEmitStart, 1, true))
	92	+	select {
	93	+	case <-entered:
	94	+	case <-time.After(time.Second):
	95	+	t.Fatal("observer did not start")
	96	+	}
	97	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeComposeStart, 1, true))
	98	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeComposeEnd, 1, true))
	99	+	observer.ObserveRuntime(NewRuntimeMark("client", RuntimeEmitEnd, 1, true)) // bounded loss
	100	+	close(release)
	101	+	observer.Flush()
	102	+	observer.Close()
	103	+
	104	+	mu.Lock()
	105	+	defer mu.Unlock()
	106	+	if len(got) != 4 \|\| got[0].Kind != RuntimeEmitStart \|\| got[1].Kind != RuntimeComposeStart \|\| got[2].Kind != RuntimeComposeEnd \|\| got[3].Kind != RuntimeTransportDiagnostic \|\| got[3].Bytes != 1 \|\| got[3].Valid {
	107	+	t.Fatalf("bounded-loss marks = %#v", got)
	108	+	}
	109	+	}
	110	+
	111	+	func TestRuntimeObserverRequiredSpanKinds(t *testing.T) {
	112	+	pairs := []struct {
	113	+	start RuntimeMarkKind
	114	+	end RuntimeMarkKind
	115	+	}{
	116	+	{RuntimeDiffStart, RuntimeDiffEnd},
	117	+	{RuntimeQueueEnqueued, RuntimeQueueDequeued},
	118	+	{RuntimeACKBlockedStart, RuntimeACKBlockedEnd},
	119	+	{RuntimeAdapterSendStart, RuntimeAdapterSendEnd},
	120	+	{RuntimeAdapterReceiveStart, RuntimeAdapterReceiveEnd},
	121	+	}
	122	+	for _, pair := range pairs {
	123	+	if pair.start == "" \|\| pair.end == "" \|\| pair.start == pair.end {
	124	+	t.Fatalf("invalid required span mapping %#v", pair)
	125	+	}
	126	+	}
	127	+	}
	128	+
	129	+	func TestRuntimeCorrelationObserverUsesHarnessProcessInputs(t *testing.T) {
	130	+	var got []RuntimeMark
	131	+	observer, err := NewRuntimeCorrelationObserver(runtimeObserverFunc(func(mark RuntimeMark) {
	132	+	got = append(got, mark)
	133	+	}), RuntimeCorrelationInputs{Scenario: "1x4-idle-local", Run: 7})
	134	+	if err != nil {
	135	+	t.Fatal(err)
	136	+	}
	137	+
	138	+	observer.ObserveRuntime(RuntimeMark{Schema: RuntimeMarkSchema, Component: "daemon", Scenario: "runtime", Run: 1, Kind: RuntimeEmitEnd, Valid: true})
	139	+	observer.ObserveRuntime(RuntimeMark{Schema: RuntimeMarkSchema, Component: "daemon", Scenario: "runtime", Run: 1, Sequence: 9, RequestID: 10, Epoch: 11, Kind: RuntimeEmitStart, Valid: true})
	140	+
	141	+	if len(got) != 2 {
	142	+	t.Fatalf("marks=%+v", got)
	143	+	}
	144	+	for _, mark := range got {
	145	+	if mark.Scenario != "1x4-idle-local" \|\| mark.Run != 7 {
	146	+	t.Fatalf("harness correlation was not applied: %+v", mark)
	147	+	}
	148	+	if mark.Sequence == 0 \|\| mark.RequestID == 0 \|\| mark.Epoch == 0 {
	149	+	t.Fatalf("missing deterministic operation identity: %+v", mark)
	150	+	}
	151	+	}
	152	+	if got[1].Sequence != 9 \|\| got[1].RequestID != 10 \|\| got[1].Epoch != 11 {
	153	+	t.Fatalf("explicit operation identity changed: %+v", got[1])
	154	+	}
	155	+	}
	156	+
	157	+	func TestRuntimeCorrelationObserverAllocatesMissingIDsProcessWide(t *testing.T) {
	158	+	var got []RuntimeMark
	159	+	collect := runtimeObserverFunc(func(mark RuntimeMark) { got = append(got, mark) })
	160	+	first, err := NewRuntimeCorrelationObserver(collect, RuntimeCorrelationInputs{Scenario: "scenario", Run: 1})
	161	+	if err != nil {
	162	+	t.Fatal(err)
	163	+	}
	164	+	second, err := NewRuntimeCorrelationObserver(collect, RuntimeCorrelationInputs{Scenario: "scenario", Run: 1})
	165	+	if err != nil {
	166	+	t.Fatal(err)
	167	+	}
	168	+
	169	+	first.ObserveRuntime(RuntimeMark{Schema: RuntimeMarkSchema, Component: "first", Kind: RuntimeEmitStart, Valid: true})
	170	+	second.ObserveRuntime(RuntimeMark{Schema: RuntimeMarkSchema, Component: "second", Kind: RuntimeEmitEnd, Valid: true})
	171	+
	172	+	if len(got) != 2 {
	173	+	t.Fatalf("marks=%+v", got)
	174	+	}
	175	+	if got[0].Sequence == got[1].Sequence {
	176	+	t.Fatalf("wrapper-local sequence collision: %+v", got)
	177	+	}
	178	+	for _, mark := range got {
	179	+	if mark.RequestID != mark.Sequence \|\| mark.Epoch != mark.RequestID {
	180	+	t.Fatalf("missing-ID fallback changed: %+v", mark)
	181	+	}
	182	+	}
	183	+	}
	184	+
	185	+	func TestRuntimeCorrelationObserverRejectsInvalidHarnessInputs(t *testing.T) {
	186	+	for _, in := range []RuntimeCorrelationInputs{{}, {Scenario: "scenario"}} {
	187	+	if _, err := NewRuntimeCorrelationObserver(runtimeObserverFunc(func(RuntimeMark) {}), in); err == nil {
	188	+	t.Fatalf("invalid inputs accepted: %+v", in)
	189	+	}
	190	+	}
	191	+	}
	192	+
	193	+	func TestRuntimeMarkHasNoRenderingPayloadOrPolicyFields(t *testing.T) {
	194	+	typ := reflect.TypeFor[RuntimeMark]()
	195	+	for _, forbidden := range []string{"Cell", "Cells", "Payload", "Frame", "Renderer", "Policy", "Transport"} {
	196	+	if _, ok := typ.FieldByName(forbidden); ok {
	197	+	t.Fatalf("RuntimeMark must not carry %s", forbidden)
	198	+	}
	199	+	}
	200	+	}
	201	+
	202	+	type runtimeObserverFunc func(RuntimeMark)
	203	+
	204	+	func (f runtimeObserverFunc) ObserveRuntime(mark RuntimeMark) { f(mark) }

M internal/usecase/client/client.go +52 -5


@@ -102,6 +102,10 @@ const stdinBufSize = 4096
102		// goroutine, decoupling the input/resize pumps from transport back-pressure.	102		// goroutine, decoupling the input/resize pumps from transport back-pressure.
103		const sendQueueDepth = 64	103		const sendQueueDepth = 64
104			104
			105	+	// runtimeObserverQueueDepth bounds marks awaiting the one reporting worker.
			106	+	// A full queue records an ordered diagnostic gap rather than blocking ACKs.
			107	+	const runtimeObserverQueueDepth = 64
			108	+
105		var reconnectSleep = sleepReconnect	109		var reconnectSleep = sleepReconnect
106		var reconnectSleepWithResize = sleepReconnectWithResizeEvents	110		var reconnectSleepWithResize = sleepReconnectWithResizeEvents
107			111

@@ -120,7 +124,36 @@ const (
120		// ClipboardReader configured (clipboard non-nil) — local attaches forward	124		// ClipboardReader configured (clipboard non-nil) — local attaches forward
121		// 0x16 untouched so the locally running agent's own clipboard handling	125		// 0x16 untouched so the locally running agent's own clipboard handling
122		// applies.	126		// applies.
123	-	func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger) (retErr error) {	127	+	// Option configures opt-in client runtime observation.
			128	+	type Option func(*runtimeOptions)
			129	+
			130	+	type runtimeOptions struct{ observer ports.RuntimeObserver }
			131	+
			132	+	// WithRuntimeObserver enables process-local marks. It intentionally accepts no
			133	+	// clock; timestamps are assigned by the concrete observer.
			134	+	func WithRuntimeObserver(observer ports.RuntimeObserver) Option {
			135	+	return func(opts *runtimeOptions) { opts.observer = observer }
			136	+	}
			137	+
			138	+	func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger) error {
			139	+	return run(ctx, dialer, term, clk, intent, name, remote, clipboard, log, nil)
			140	+	}
			141	+
			142	+	// RunWithRuntimeObserver is the application wiring entry point.
			143	+	func RunWithRuntimeObserver(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger, observer ports.RuntimeObserver) error {
			144	+	return run(ctx, dialer, term, clk, intent, name, remote, clipboard, log, observer)
			145	+	}
			146	+
			147	+	func run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk ports.Clock, intent uint8, name string, remote bool, clipboard ports.ClipboardReader, log *slog.Logger, observer ports.RuntimeObserver) (retErr error) {
			148	+	if reporter, owned := ports.EnsureSerializedRuntimeObserver(observer, runtimeObserverQueueDepth); reporter != nil {
			149	+	if owned {
			150	+	defer func() {
			151	+	reporter.Flush()
			152	+	reporter.Close()
			153	+	}()
			154	+	}
			155	+	observer = reporter
			156	+	}
124		if log == nil {	157		if log == nil {
125		log = slog.Default()	158		log = slog.Default()
126		}	159		}

@@ -244,7 +277,7 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port
244		if reporter, ok := transport.(ports.LinkStateReporter); ok {	277		if reporter, ok := transport.(ports.LinkStateReporter); ok {
245		linkEvents = reporter.LinkEvents()	278		linkEvents = reporter.LinkEvents()
246		}	279		}
247	-	result := attachOnce(ctx, transport, term, clk, attemptIntent, attemptName, resumeToken, processClientID, &ms, enterRaw, clearStatus, drawStatusStage, linkEvents, remote, clipboard, log)	280	+	result := attachOnce(ctx, transport, term, clk, attemptIntent, attemptName, resumeToken, processClientID, &ms, enterRaw, clearStatus, drawStatusStage, linkEvents, remote, clipboard, log, observer)
248		if result.welcomed {	281		if result.welcomed {
249		backoff = defaultReconnectBackoff.initial	282		backoff = defaultReconnectBackoff.initial
250		}	283		}

@@ -311,8 +344,14 @@ func sleepReconnectWithResizeEvents(ctx context.Context, clk ports.Clock, d time
311		// disappears, or the context is cancelled.	344		// disappears, or the context is cancelled.
312		//	345		//
313		// It is kept as a compatibility wrapper for callers that still own dialing.	346		// It is kept as a compatibility wrapper for callers that still own dialing.
314	-	func Attach(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string) error {	347	+	func Attach(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, options ...Option) error {
315	-	return Run(ctx, singleTransportDialer{transport: transport}, term, clk, intent, name, false, nil, slog.Default())	348	+	var opts runtimeOptions
			349	+	for _, option := range options {
			350	+	if option != nil {
			351	+	option(&opts)
			352	+	}
			353	+	}
			354	+	return run(ctx, singleTransportDialer{transport: transport}, term, clk, intent, name, false, nil, slog.Default(), opts.observer)
316		}	355		}
317			356
318		type singleTransportDialer struct{ transport ports.Transport }	357		type singleTransportDialer struct{ transport ports.Transport }

@@ -346,7 +385,11 @@ func requestedOutputWindow(transport ports.Transport) uint8 {
346		return 8	385		return 8
347		}	386		}
348			387
349	-	func attachOnce(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, resumeToken uint64, clientID [16]byte, ms milestones, enterRaw func() error, clearStatus func(), drawStatusStage func(reconnectStage), linkEvents <-chan ports.LinkEvent, remote bool, clipboard ports.ClipboardReader, log slog.Logger) attachResult {	388	+	func attachOnce(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, resumeToken uint64, clientID [16]byte, ms milestones, enterRaw func() error, clearStatus func(), drawStatusStage func(reconnectStage), linkEvents <-chan ports.LinkEvent, remote bool, clipboard ports.ClipboardReader, log slog.Logger, observers ...ports.RuntimeObserver) attachResult {
			389	+	var observer ports.RuntimeObserver
			390	+	if len(observers) != 0 {
			391	+	observer = observers[0]
			392	+	}
350		// 1. Handshake: send Hello with our size and TERM.	393		// 1. Handshake: send Hello with our size and TERM.
351		size, err := term.Size()	394		size, err := term.Size()
352		if err != nil {	395		if err != nil {

@@ -501,6 +544,10 @@ func attachOnce(ctx context.Context, transport ports.Transport, term ports.Termi
501		if ferr := term.Flush(); ferr != nil {	544		if ferr := term.Flush(); ferr != nil {
502		return welcomedResult(fmt.Errorf("vev: flushing terminal: %w", ferr))	545		return welcomedResult(fmt.Errorf("vev: flushing terminal: %w", ferr))
503		}	546		}
			547	+	// The terminal boundary is after a successful flush and before ACK.
			548	+	if observer != nil {
			549	+	observer.ObserveRuntime(ports.NewRuntimeMark("client", ports.RuntimeTerminalFlushed, uint64(len(o.Data)), true))
			550	+	}
504		if o.NewStateNum != 0 {	551		if o.NewStateNum != 0 {
505		ackQueue.offer(o.NewStateNum)	552		ackQueue.offer(o.NewStateNum)
506		}	553		}

A internal/usecase/client/transport_observability_test.go +140 -0

@@ -0,0 +1,140 @@
	1	+	package client_test
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"context"
	6	+	"sync"
	7	+	"sync/atomic"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/stretchr/testify/require"
	12	+
	13	+	"github.com/bnema/vev/internal/domain"
	14	+	"github.com/bnema/vev/internal/ports"
	15	+	portsmocks "github.com/bnema/vev/internal/ports/mocks"
	16	+	"github.com/bnema/vev/internal/usecase/client"
	17	+	)
	18	+
	19	+	func TestBlockingRuntimeObserverDoesNotDelayTerminalFlushOrACK(t *testing.T) {
	20	+	var out bytes.Buffer
	21	+	var restores atomic.Int32
	22	+	resizeCh := make(chan domain.Size)
	23	+	flushed := make(chan struct{})
	24	+	term := portsmocks.NewMockTerminal(t)
	25	+	term.EXPECT().Size().Return(domain.Size{Cols: 80, Rows: 24}, nil).Once()
	26	+	term.EXPECT().EnterRaw().Return(func() error { restores.Add(1); return nil }, nil).Once()
	27	+	in := newBlockingReader()
	28	+	defer in.unblock()
	29	+	term.EXPECT().In().Return(in).Maybe()
	30	+	term.EXPECT().Out().Return(&out).Maybe()
	31	+	term.EXPECT().Flush().Run(func() { close(flushed) }).Return(nil).Once()
	32	+	term.EXPECT().ResizeEvents().Return(resizeCh).Maybe()
	33	+
	34	+	observer := &blockingClientRuntimeObserver{entered: make(chan struct{}), release: make(chan struct{})}
	35	+	acked := make(chan struct{})
	36	+	transport := portsmocks.NewMockTransport(t)
	37	+	transport.EXPECT().Send(isType(ports.MsgHello)).Return(nil).Once()
	38	+	transport.EXPECT().Send(isType(ports.MsgAck)).Run(func(ports.Frame) { close(acked) }).Return(nil).Once()
	39	+	unblock := scriptRecv(transport,
	40	+	recvItem{f: frameOf(ports.MsgWelcome, ports.MarshalWelcome(ports.Welcome{SessionID: "s1"}))},
	41	+	recvItem{f: frameOf(ports.MsgOutput, ports.MarshalOutput(ports.Output{Data: []byte("flush-before-observe"), NewStateNum: 3}))},
	42	+	)
	43	+	defer unblock()
	44	+	transport.EXPECT().Close().Return(nil).Once()
	45	+
	46	+	ctx, cancel := context.WithCancel(context.Background())
	47	+	result := make(chan error, 1)
	48	+	go func() {
	49	+	result <- client.Attach(ctx, transport, term, realClock{}, ports.IntentEphemeral, "", client.WithRuntimeObserver(observer))
	50	+	}()
	51	+	awaitClientRuntime(t, observer.entered, "blocking observer")
	52	+	awaitClientRuntime(t, flushed, "terminal flush")
	53	+	awaitClientRuntime(t, acked, "ACK")
	54	+	cancel()
	55	+	close(observer.release)
	56	+	select {
	57	+	case err := <-result:
	58	+	require.NoError(t, err)
	59	+	case <-time.After(time.Second):
	60	+	t.Fatal("Attach did not complete after observer release")
	61	+	}
	62	+	require.Equal(t, "flush-before-observe", out.String())
	63	+	require.Equal(t, int32(1), restores.Load())
	64	+	}
	65	+
	66	+	func TestTerminalFlushBoundaryTransportObservability(t *testing.T) {
	67	+	var out bytes.Buffer
	68	+	var restores atomic.Int32
	69	+	resizeCh := make(chan domain.Size)
	70	+	term, in := newHappyTerminal(t, &out, &restores, resizeCh)
	71	+	defer in.unblock()
	72	+
	73	+	observer := &clientRuntimeObserver{}
	74	+	transport := portsmocks.NewMockTransport(t)
	75	+	transport.EXPECT().Send(isType(ports.MsgHello)).Return(nil).Once()
	76	+	unblock := scriptRecv(transport,
	77	+	recvItem{f: frameOf(ports.MsgWelcome, ports.MarshalWelcome(ports.Welcome{SessionID: "s1"}))},
	78	+	recvItem{f: frameOf(ports.MsgOutput, ports.MarshalOutput(ports.Output{Data: []byte("unchanged-by-observer"), NewStateNum: 3}))},
	79	+	recvItem{f: frameOf(ports.MsgDetached, ports.MarshalDetached(ports.Detached{Reason: ports.ReasonDetach}))},
	80	+	)
	81	+	defer unblock()
	82	+	transport.EXPECT().Send(isType(ports.MsgAck)).Return(nil).Maybe()
	83	+	transport.EXPECT().Close().Return(nil).Once()
	84	+
	85	+	err := client.Attach(context.Background(), transport, term, realClock{}, ports.IntentEphemeral, "", client.WithRuntimeObserver(observer))
	86	+	require.NoError(t, err)
	87	+	require.Equal(t, "unchanged-by-observer", out.String(), "observer must not alter terminal bytes")
	88	+	require.Equal(t, int32(1), restores.Load())
	89	+	// Carriage spans belong only to concrete adapters. The client owns the
	90	+	// post-successful-flush terminal boundary, before its ACK is queued.
	91	+	observer.requireOrdered(t, ports.RuntimeTerminalFlushed)
	92	+	}
	93	+
	94	+	type blockingClientRuntimeObserver struct {
	95	+	entered chan struct{}
	96	+	release chan struct{}
	97	+	once sync.Once
	98	+	}
	99	+
	100	+	func (o *blockingClientRuntimeObserver) ObserveRuntime(ports.RuntimeMark) {
	101	+	o.once.Do(func() {
	102	+	close(o.entered)
	103	+	<-o.release
	104	+	})
	105	+	}
	106	+
	107	+	func awaitClientRuntime(t *testing.T, ch <-chan struct{}, what string) {
	108	+	t.Helper()
	109	+	select {
	110	+	case <-ch:
	111	+	case <-time.After(time.Second):
	112	+	t.Fatalf("timed out waiting for %s", what)
	113	+	}
	114	+	}
	115	+
	116	+	type clientRuntimeObserver struct {
	117	+	mu sync.Mutex
	118	+	marks []ports.RuntimeMark
	119	+	}
	120	+
	121	+	func (o *clientRuntimeObserver) ObserveRuntime(mark ports.RuntimeMark) {
	122	+	o.mu.Lock()
	123	+	defer o.mu.Unlock()
	124	+	o.marks = append(o.marks, mark)
	125	+	}
	126	+
	127	+	func (o clientRuntimeObserver) requireOrdered(t testing.T, kinds ...ports.RuntimeMarkKind) {
	128	+	t.Helper()
	129	+	o.mu.Lock()
	130	+	defer o.mu.Unlock()
	131	+	next := 0
	132	+	for _, mark := range o.marks {
	133	+	if next < len(kinds) && mark.Kind == kinds[next] {
	134	+	next++
	135	+	}
	136	+	}
	137	+	if next != len(kinds) {
	138	+	t.Fatalf("runtime marks %#v do not contain ordered %#v", o.marks, kinds)
	139	+	}
	140	+	}

M internal/usecase/daemon/client.go +2 -1

 	rc := sess.renderCoordinator()
 	if rc == nil {
 		rc = newRenderCoordinator(renderCoordinatorOptions{
-			clock: d.clock,
+			clock:    d.clock,
+			observer: d.runtimeObserver,
 			wake: func(w renderWake) {
 				// Never reread sess.client here: a wake is bound to both its
 				// attachment and its coordinator incarnation. The second check in

M internal/usecase/daemon/daemon.go +59 -21

 // Scheduler debounce bounds. Idle updates use the minimum for low latency;
 // sustained floods step toward the maximum to reduce frame/syscall pressure.
 const (
-	minDebounceInterval   = 2 * time.Millisecond
+	minDebounceInterval       = 2 * time.Millisecond
-	maxSyncUpdateDuration = 500 * time.Millisecond
+	maxSyncUpdateDuration     = 500 * time.Millisecond
+	runtimeObserverQueueDepth = 64
 )
 const debounceInterval = minDebounceInterval
 	// interval between JRS validation and its committed hand-off.
 	beforeRecentSessionHandoff func()
-	ptys                    ports.PTYFactory
+	ptys                     ports.PTYFactory
-	clock                   ports.Clock
+	clock                    ports.Clock
-	log                     *slog.Logger
+	log                      *slog.Logger
-	baseEnv                 []string
+	runtimeObserver          ports.RuntimeObserver
-	shell                   string
+	runtimeObserverCloser    ports.SerializedRuntimeObserver
-	shellArgs               []string
+	runtimeObserverCloseOnce sync.Once
-	persist                 *persist.Persister
+	baseEnv                  []string
-	persistEnabled          bool
+	shell                    string
-	snaps                   ports.SnapshotStore
+	shellArgs                []string
-	snapsEnabled            bool
+	persist                  *persist.Persister
-	snapshotMarshal         func(snapcodec.Session) ([]byte, error)
+	persistEnabled           bool
-	snapshotJobs            chan *snapshotCapture
+	snaps                    ports.SnapshotStore
-	snapshotWorkerMu        sync.Mutex
+	snapsEnabled             bool
-	snapshotWorkerID        uint64
+	snapshotMarshal          func(snapcodec.Session) ([]byte, error)
-	snapshotWorkerCtx       context.Context
+	snapshotJobs             chan *snapshotCapture
-	snapshotWorkerCancel    context.CancelFunc
+	snapshotWorkerMu         sync.Mutex
-	snapshotWorkerDone      chan struct{}
+	snapshotWorkerID         uint64
-	snapshotWorkerFlush     chan struct{}
+	snapshotWorkerCtx        context.Context
-	snapshotWorkerFinalWake chan struct{}
+	snapshotWorkerCancel     context.CancelFunc
+	snapshotWorkerDone       chan struct{}
+	snapshotWorkerFlush      chan struct{}
+	snapshotWorkerFinalWake  chan struct{}
 	// snapshotFinalJobs retains immutable terminal captures rejected by the
 	// bounded regular queue. It is only populated during session teardown and
 	// drained by the same worker, so persistence remains outside daemon locks.
 type Option func(*Daemon)
+// WithRuntimeObserver enables opt-in process-local marks. It takes no clock:
+// the concrete observer is the sole timestamp owner.
+func WithRuntimeObserver(observer ports.RuntimeObserver) Option {
+	return func(d *Daemon) {
+		// Options are applied before the daemon starts. Replacing an owned
+		// observer here cannot race producers and must retire its worker.
+		if d.runtimeObserverCloser != nil {
+			d.runtimeObserverCloser.Close()
+			d.runtimeObserverCloser = nil
+		}
+		reporter, owned := ports.EnsureSerializedRuntimeObserver(observer, runtimeObserverQueueDepth)
+		d.runtimeObserver = reporter
+		if owned {
+			d.runtimeObserverCloser = reporter
+		}
+	}
+}
+// closeRuntimeObserver drains the daemon-owned reporter before teardown
+// completes. Application-owned process reporters are closed by their app
+// lifecycle instead, so shared observers never receive a duplicate close.
+func (d *Daemon) closeRuntimeObserver() {
+	if d == nil {
+		return
+	}
+	d.runtimeObserverCloseOnce.Do(func() {
+		if d.runtimeObserverCloser != nil {
+			d.runtimeObserverCloser.Flush()
+			d.runtimeObserverCloser.Close()
+		}
+	})
+}
 // WithShell overrides the command (and its args) each session spawns. The
 // default is $SHELL (or /bin/sh) with no arguments; tests use this to run a
 // deterministic program.
 // returns when the last session is removed or ctx is cancelled; on the latter
 // path attached clients are detached with ReasonServerShutdown.
 func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error {
+	defer d.closeRuntimeObserver()
 	d.serveCtx, d.serveCancel = context.WithCancel(ctx)
 	defer d.serveCancel()
 	d.hardCtx, d.hardCancel = context.WithCancel(context.Background())

M internal/usecase/daemon/render.go +56 -2


@@ -5,6 +5,7 @@ import (
5		"bytes"	5		"bytes"
6			6
7		"github.com/bnema/vev/internal/domain"	7		"github.com/bnema/vev/internal/domain"
			8	+	"github.com/bnema/vev/internal/ports"
8		"github.com/bnema/vev/internal/usecase/layout"	9		"github.com/bnema/vev/internal/usecase/layout"
9		themeui "github.com/bnema/vev/internal/usecase/theme"	10		themeui "github.com/bnema/vev/internal/usecase/theme"
10		"github.com/bnema/vev/pkg/renderer"	11		"github.com/bnema/vev/pkg/renderer"

@@ -193,7 +194,50 @@ func (d Daemon) flushPTYEffects(sess session, tb tab, p pane) {
193		// paint draws the composed client frame (active tab plus status bar) and	194		// paint draws the composed client frame (active tab plus status bar) and
194		// sends the resulting bytes. The renderer shadow is reset on explicit invalidations	195		// sends the resulting bytes. The renderer shadow is reset on explicit invalidations
195		// such as switch/create/close/rename/resize so the repaint is complete.	196		// such as switch/create/close/rename/resize so the repaint is complete.
			197	+	func (d *Daemon) observeRuntime(kind ports.RuntimeMarkKind, bytes uint64, valid bool) {
			198	+	if d != nil && d.runtimeObserver != nil {
			199	+	d.runtimeObserver.ObserveRuntime(ports.NewRuntimeMark("daemon", kind, bytes, valid))
			200	+	}
			201	+	}
			202	+
			203	+	// runtimeMarkBatch captures immutable runtime marks during a render transaction.
			204	+	// paint owns attachment sendMu while it captures, composes, and emits; JSONL
			205	+	// observer I/O must therefore happen only after that ownership is released.
			206	+	type runtimeMarkBatch struct {
			207	+	observer ports.RuntimeObserver
			208	+	marks []ports.RuntimeMark
			209	+	}
			210	+
			211	+	func (d *Daemon) newRuntimeMarkBatch() runtimeMarkBatch {
			212	+	if d == nil {
			213	+	return runtimeMarkBatch{}
			214	+	}
			215	+	return runtimeMarkBatch{observer: d.runtimeObserver}
			216	+	}
			217	+
			218	+	func (b *runtimeMarkBatch) span(start, end ports.RuntimeMarkKind, bytes uint64) func(uint64, bool) {
			219	+	if b == nil \|\| b.observer == nil {
			220	+	return func(uint64, bool) {}
			221	+	}
			222	+	correlation := ports.NewRuntimeCorrelation()
			223	+	b.marks = append(b.marks, ports.NewRuntimeMarkWithCorrelation("daemon", correlation, start, bytes, true))
			224	+	return func(endBytes uint64, valid bool) {
			225	+	b.marks = append(b.marks, ports.NewRuntimeMarkWithCorrelation("daemon", correlation, end, endBytes, valid))
			226	+	}
			227	+	}
			228	+
			229	+	func (b *runtimeMarkBatch) flush() {
			230	+	if b == nil \|\| b.observer == nil {
			231	+	return
			232	+	}
			233	+	for _, mark := range b.marks {
			234	+	b.observer.ObserveRuntime(mark)
			235	+	}
			236	+	}
			237	+
196		func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...uint64) {	238		func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...uint64) {
			239	+	marks := d.newRuntimeMarkBatch()
			240	+
197		attachmentEpoch := uint64(0)	241		attachmentEpoch := uint64(0)
198		if len(epochs) != 0 {	242		if len(epochs) != 0 {
199		attachmentEpoch = epochs[0]	243		attachmentEpoch = epochs[0]

@@ -218,6 +262,8 @@ func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...
218		// the normal gate, but this ownership check also protects direct resize and	262		// the normal gate, but this ownership check also protects direct resize and
219		// test paint paths from consuming damage that cannot be emitted.	263		// test paint paths from consuming damage that cannot be emitted.
220		if ac.output != nil && ac.output.atCapacity() {	264		if ac.output != nil && ac.output.atCapacity() {
			265	+	// The coordinator owns the blocked interval; this guard only protects
			266	+	// direct test/resize paint calls from destructive capture.
221		ac.sendMu.Unlock()	267		ac.sendMu.Unlock()
222		return	268		return
223		}	269		}

@@ -228,11 +274,15 @@ func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...
228		overlays := ac.overlays.SnapshotForRender()	274		overlays := ac.overlays.SnapshotForRender()
229		repaintAttachedClients := false	275		repaintAttachedClients := false
230		defer func() {	276		defer func() {
			277	+	overlays.Unlock()
			278	+	// Every return below has released sendMu before reaching this defer.
			279	+	// Emit the captured sequence before any follow-up repaint can introduce
			280	+	// another boundary, while no attachment, session, tab, or pane lock is held.
			281	+	marks.flush()
231		if repaintAttachedClients {	282		if repaintAttachedClients {
232		d.repaintAllAttachedClients()	283		d.repaintAllAttachedClients()
233		}	284		}
234		}()	285		}()
235	-	defer overlays.Unlock()
236		preview := snapshotPickerPreview(nil)	286		preview := snapshotPickerPreview(nil)
237		if overlays.previewTab != tb {	287		if overlays.previewTab != tb {
238		preview = snapshotPickerPreview(overlays.previewTab)	288		preview = snapshotPickerPreview(overlays.previewTab)

@@ -270,7 +320,9 @@ func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...
270		pickerActive: overlays.pickerActive, paletteActive: overlays.paletteActive, promptActive: overlays.promptActive,	320		pickerActive: overlays.pickerActive, paletteActive: overlays.paletteActive, promptActive: overlays.promptActive,
271		copyFeedback: overlays.copyFeedback,	321		copyFeedback: overlays.copyFeedback,
272		}	322		}
			323	+	endCapture := marks.span(ports.RuntimeCaptureStart, ports.RuntimeCaptureEnd, 0)
273		state, ok := captureRenderState(sess, ac, bars, capturedOverlays, preview, floatingCfg, reset, damageCaptureConsume)	324		state, ok := captureRenderState(sess, ac, bars, capturedOverlays, preview, floatingCfg, reset, damageCaptureConsume)
			325	+	endCapture(0, ok)
274		if !ok {	326		if !ok {
275		ac.sendMu.Unlock()	327		ac.sendMu.Unlock()
276		return	328		return

@@ -285,11 +337,13 @@ func (d Daemon) paint(sess session, ac *attachedClient, reset bool, epochs ...
285		state.preview = pickerPreviewFromCapturedRender(previewState)	337		state.preview = pickerPreviewFromCapturedRender(previewState)
286		}	338		}
287		captureOverlayLayers(state, overlays, paletteCfg)	339		captureOverlayLayers(state, overlays, paletteCfg)
			340	+	endCompose := marks.span(ports.RuntimeComposeStart, ports.RuntimeComposeEnd, 0)
288		composed := composeFrame(*state, ac.pipelineCache, ac.pipelineScratch)	341		composed := composeFrame(*state, ac.pipelineCache, ac.pipelineScratch)
			342	+	endCompose(0, true)
289		if ac.renderStages.compose != nil {	343		if ac.renderStages.compose != nil {
290		ac.renderStages.compose()	344		ac.renderStages.compose()
291		}	345		}
292	-	d.emitFrame(sess, ac, state, composed)	346	+	d.emitFrame(sess, ac, state, composed, &marks)
293		}	347		}
294			348
295		type themeStyles struct {	349		type themeStyles struct {

M internal/usecase/daemon/render_coordinator.go +131 -1


@@ -78,7 +78,8 @@ type resizeRequestMetadata struct {
78			78
79		// renderCoordinatorOptions wires one coordinator instance.	79		// renderCoordinatorOptions wires one coordinator instance.
80		type renderCoordinatorOptions struct {	80		type renderCoordinatorOptions struct {
81	-	clock ports.Clock	81	+	clock ports.Clock
			82	+	observer ports.RuntimeObserver
82		// wake is the transitional composition target.	83		// wake is the transitional composition target.
83		wake func(renderWake)	84		wake func(renderWake)
84		// ackReady reports whether the attachment may compose another output	85		// ackReady reports whether the attachment may compose another output

@@ -107,6 +108,56 @@ type renderCoordinatorBurstMetrics struct {
107		coalesced atomic.Uint64	108		coalesced atomic.Uint64
108		}	109		}
109			110
			111	+	// ackBlockedSpan hands an ACK-blocked endpoint pair between coordinator
			112	+	// transitions without holding c.mu while the observer runs. A consuming ACK
			113	+	// may arrive while Start is blocked in observer I/O; finish records that
			114	+	// closure and publishStart emits it only after Start has returned.
			115	+	type ackBlockedSpan struct {
			116	+	observer ports.RuntimeObserver
			117	+	correlation ports.RuntimeCorrelation
			118	+
			119	+	mu sync.Mutex
			120	+	startPublished bool
			121	+	finished bool
			122	+	endPending bool
			123	+	endValid bool
			124	+	}
			125	+
			126	+	func newACKBlockedSpan(observer ports.RuntimeObserver) *ackBlockedSpan {
			127	+	return &ackBlockedSpan{observer: observer, correlation: ports.NewRuntimeCorrelation()}
			128	+	}
			129	+
			130	+	func (s *ackBlockedSpan) publishStart() {
			131	+	s.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", s.correlation, ports.RuntimeACKBlockedStart, 0, true))
			132	+
			133	+	s.mu.Lock()
			134	+	s.startPublished = true
			135	+	endPending, endValid := s.endPending, s.endValid
			136	+	s.endPending = false
			137	+	s.mu.Unlock()
			138	+	if endPending {
			139	+	s.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", s.correlation, ports.RuntimeACKBlockedEnd, 0, endValid))
			140	+	}
			141	+	}
			142	+
			143	+	// finish closes a span at most once. It never invokes the observer before the
			144	+	// blocked start has completed publication.
			145	+	func (s *ackBlockedSpan) finish(valid bool) {
			146	+	s.mu.Lock()
			147	+	if s.finished {
			148	+	s.mu.Unlock()
			149	+	return
			150	+	}
			151	+	s.finished = true
			152	+	if !s.startPublished {
			153	+	s.endPending, s.endValid = true, valid
			154	+	s.mu.Unlock()
			155	+	return
			156	+	}
			157	+	s.mu.Unlock()
			158	+	s.observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", s.correlation, ports.RuntimeACKBlockedEnd, 0, valid))
			159	+	}
			160	+
110		// renderCoordinatorBurstMetricsSnapshot is an immutable, internal view for	161		// renderCoordinatorBurstMetricsSnapshot is an immutable, internal view for
111		// benchmark reporting. Scheduling only writes the atomic counters above.	162		// benchmark reporting. Scheduling only writes the atomic counters above.
112		type renderCoordinatorBurstMetricsSnapshot struct {	163		type renderCoordinatorBurstMetricsSnapshot struct {

@@ -124,6 +175,10 @@ type renderCoordinator struct {
124		pending bool	175		pending bool
125		pendingReset bool	176		pendingReset bool
126		pendingUrgent bool	177		pendingUrgent bool
			178	+	// queueMarked spans the coordinator's actual pending interval, from first
			179	+	// enqueue until fire consumes the coalesced work.
			180	+	queueMarked bool
			181	+	queueCorrelation ports.RuntimeCorrelation
127		// pendingPreview is accounted separately from the target primary wake:	182		// pendingPreview is accounted separately from the target primary wake:
128		// viewers may consume a coalesced target snapshot while that target is	183		// viewers may consume a coalesced target snapshot while that target is
129		// still ACK-blocked.	184		// still ACK-blocked.

@@ -131,6 +186,9 @@ type renderCoordinator struct {
131		// ackDeferred records that an expired deadline or watchdog was blocked by	186		// ackDeferred records that an expired deadline or watchdog was blocked by
132		// output-window capacity. ACK notifications may flush only this state.	187		// output-window capacity. ACK notifications may flush only this state.
133		ackDeferred bool	188		ackDeferred bool
			189	+	// ackBlocked retains the first rejected ACK-capacity probe until the
			190	+	// matching consume or lifecycle closure releases that exact interval.
			191	+	ackBlocked *ackBlockedSpan
134		// deadlineDue closes the handoff between a timer worker receiving its tick	192		// deadlineDue closes the handoff between a timer worker receiving its tick
135		// and fire observing unavailable ACK capacity.	193		// and fire observing unavailable ACK capacity.
136		deadlineDue bool	194		deadlineDue bool

@@ -287,9 +345,17 @@ func (c renderCoordinator) invalidateForAttachmentAtResizeEpoch(source attache
287		onInvalidate := c.opts.onInvalidate	345		onInvalidate := c.opts.onInvalidate
288		c.metrics.invalidations.Add(1)	346		c.metrics.invalidations.Add(1)
289		wasPending, wasUrgent, wasPreviewPending := c.pending, c.pendingUrgent, c.pendingPreview	347		wasPending, wasUrgent, wasPreviewPending := c.pending, c.pendingUrgent, c.pendingPreview
			348	+	queueStart := false
			349	+	var queueCorrelation ports.RuntimeCorrelation
290		if !wasPending {	350		if !wasPending {
291		c.ackDeferred = false	351		c.ackDeferred = false
292		c.deadlineDue = false	352		c.deadlineDue = false
			353	+	if c.opts.observer != nil {
			354	+	c.queueMarked = true
			355	+	c.queueCorrelation = ports.NewRuntimeCorrelation()
			356	+	queueCorrelation = c.queueCorrelation
			357	+	queueStart = true
			358	+	}
293		}	359		}
294		c.pending = true	360		c.pending = true
295		c.pendingReset = c.pendingReset \|\| inv.reset	361		c.pendingReset = c.pendingReset \|\| inv.reset

@@ -312,7 +378,11 @@ func (c renderCoordinator) invalidateForAttachmentAtResizeEpoch(source attache
312		}	378		}
313		clock := c.opts.clock	379		clock := c.opts.clock
314		c.armed = c.armed \|\| arm	380		c.armed = c.armed \|\| arm
			381	+	observer := c.opts.observer
315		c.mu.Unlock()	382		c.mu.Unlock()
			383	+	if queueStart && observer != nil {
			384	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueEnqueued, 0, true))
			385	+	}
316		stopTimer(old)	386		stopTimer(old)
317		if !arm \|\| clock == nil {	387		if !arm \|\| clock == nil {
318		if onInvalidate != nil {	388		if onInvalidate != nil {

@@ -377,7 +447,12 @@ func (c *renderCoordinator) notifyAck() {
377		return	447		return
378		}	448		}
379		gen := c.generation	449		gen := c.generation
			450	+	blocked := c.ackBlocked
			451	+	c.ackBlocked = nil
380		c.mu.Unlock()	452		c.mu.Unlock()
			453	+	if blocked != nil {
			454	+	blocked.finish(true)
			455	+	}
381		c.fire(gen, false, false)	456		c.fire(gen, false, false)
382		}	457		}
383			458

@@ -635,6 +710,10 @@ func (c *renderCoordinator) wakeCurrent(w renderWake) bool {
635		func (c renderCoordinator) noteDetach(ac attachedClient) {	710		func (c renderCoordinator) noteDetach(ac attachedClient) {
636		c.mu.Lock()	711		c.mu.Lock()
637		var timer, resizeTimer, retryTimer ports.Timer	712		var timer, resizeTimer, retryTimer ports.Timer
			713	+	var observer ports.RuntimeObserver
			714	+	var queueCorrelation ports.RuntimeCorrelation
			715	+	var queueMarked bool
			716	+	var ackBlocked *ackBlockedSpan
638		if c.attachment == ac {	717		if c.attachment == ac {
639		c.advanceAttachmentEpochLocked(ac)	718		c.advanceAttachmentEpochLocked(ac)
640		c.attachment = nil	719		c.attachment = nil

@@ -660,11 +739,20 @@ func (c renderCoordinator) noteDetach(ac attachedClient) {
660		}	739		}
661		retryTimer, c.retryTimer = c.retryTimer, nil	740		retryTimer, c.retryTimer = c.retryTimer, nil
662		c.retryGen++	741		c.retryGen++
			742	+	observer, queueCorrelation, queueMarked = c.opts.observer, c.queueCorrelation, c.queueMarked
			743	+	ackBlocked = c.ackBlocked
			744	+	c.queueMarked, c.ackBlocked = false, nil
663		}	745		}
664		c.mu.Unlock()	746		c.mu.Unlock()
665		stopTimer(timer)	747		stopTimer(timer)
666		stopTimer(resizeTimer)	748		stopTimer(resizeTimer)
667		stopTimer(retryTimer)	749		stopTimer(retryTimer)
			750	+	if queueMarked && observer != nil {
			751	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueDequeued, 0, false))
			752	+	}
			753	+	if ackBlocked != nil {
			754	+	ackBlocked.finish(false)
			755	+	}
668		}	756		}
669			757
670		// noteReplace hands the coordinator from old to replacement; callbacks	758		// noteReplace hands the coordinator from old to replacement; callbacks

@@ -675,6 +763,10 @@ func (c renderCoordinator) noteReplace(old, replacement attachedClient, readin
675		ready = readiness[0]	763		ready = readiness[0]
676		}	764		}
677		c.mu.Lock()	765		c.mu.Lock()
			766	+	var observer ports.RuntimeObserver
			767	+	var queueCorrelation ports.RuntimeCorrelation
			768	+	var queueMarked bool
			769	+	var ackBlocked *ackBlockedSpan
678		// A coordinator may be installed while replacing a legacy attachment	770		// A coordinator may be installed while replacing a legacy attachment
679		// which predated coordinator ownership. In that case nil has no pending	771		// which predated coordinator ownership. In that case nil has no pending
680		// identity to invalidate, and the replacement becomes the first bound	772		// identity to invalidate, and the replacement becomes the first bound

@@ -708,11 +800,20 @@ func (c renderCoordinator) noteReplace(old, replacement attachedClient, readin
708		}	800		}
709		retryTimer, c.retryTimer = c.retryTimer, nil	801		retryTimer, c.retryTimer = c.retryTimer, nil
710		c.retryGen++	802		c.retryGen++
			803	+	observer, queueCorrelation, queueMarked = c.opts.observer, c.queueCorrelation, c.queueMarked
			804	+	ackBlocked = c.ackBlocked
			805	+	c.queueMarked, c.ackBlocked = false, nil
711		}	806		}
712		c.mu.Unlock()	807		c.mu.Unlock()
713		stopTimer(timer)	808		stopTimer(timer)
714		stopTimer(resizeTimer)	809		stopTimer(resizeTimer)
715		stopTimer(retryTimer)	810		stopTimer(retryTimer)
			811	+	if queueMarked && observer != nil {
			812	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueDequeued, 0, false))
			813	+	}
			814	+	if ackBlocked != nil {
			815	+	ackBlocked.finish(false)
			816	+	}
716		}	817		}
717			818
718		// notePark invalidates pending wakes when the attachment parks for resume.	819		// notePark invalidates pending wakes when the attachment parks for resume.

@@ -747,7 +848,16 @@ func (c *renderCoordinator) noteSessionTeardown() {
747		c.armed = false	848		c.armed = false
748		timer := c.detachNormalTimerLocked()	849		timer := c.detachNormalTimerLocked()
749		timers := c.detachSyncTimersLocked()	850		timers := c.detachSyncTimersLocked()
			851	+	observer, queueCorrelation, queueMarked := c.opts.observer, c.queueCorrelation, c.queueMarked
			852	+	ackBlocked := c.ackBlocked
			853	+	c.queueMarked, c.ackBlocked = false, nil
750		c.mu.Unlock()	854		c.mu.Unlock()
			855	+	if queueMarked && observer != nil {
			856	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueDequeued, 0, false))
			857	+	}
			858	+	if ackBlocked != nil {
			859	+	ackBlocked.finish(false)
			860	+	}
751		stopTimer(timer)	861		stopTimer(timer)
752		stopTimer(resizeTimer)	862		stopTimer(resizeTimer)
753		stopTimer(retryTimer)	863		stopTimer(retryTimer)

@@ -1052,11 +1162,22 @@ func (c *renderCoordinator) fire(gen uint64, watchdog, deadline bool) {
1052		// notification rather than accumulating ackDeferred forever.	1162		// notification rather than accumulating ackDeferred forever.
1053		headlessPreviewOnly := c.detached && c.attachment == nil	1163		headlessPreviewOnly := c.detached && c.attachment == nil
1054		if !headlessPreviewOnly && (!ready \|\| (c.attachment != nil && !c.attachmentReady)) {	1164		if !headlessPreviewOnly && (!ready \|\| (c.attachment != nil && !c.attachmentReady)) {
			1165	+	var ackStart *ackBlockedSpan
1055		if !ready && deadline {	1166		if !ready && deadline {
1056		c.ackDeferred = true	1167		c.ackDeferred = true
			1168	+	if c.ackBlocked == nil && c.opts.observer != nil {
			1169	+	// Install the span while coordinator state is owned. Its Start
			1170	+	// publication runs unlocked, and its own handoff defers End until
			1171	+	// that call has returned.
			1172	+	ackStart = newACKBlockedSpan(c.opts.observer)
			1173	+	c.ackBlocked = ackStart
			1174	+	}
1057		}	1175		}
1058		preview, previews := c.takePendingPreviewsLocked()	1176		preview, previews := c.takePendingPreviewsLocked()
1059		c.mu.Unlock()	1177		c.mu.Unlock()
			1178	+	if ackStart != nil {
			1179	+	ackStart.publishStart()
			1180	+	}
1060		c.notifyPreviews(w, preview, previews)	1181		c.notifyPreviews(w, preview, previews)
1061		return	1182		return
1062		}	1183		}

@@ -1075,9 +1196,18 @@ func (c *renderCoordinator) fire(gen uint64, watchdog, deadline bool) {
1075		}	1196		}
1076		timer := c.detachNormalTimerLocked()	1197		timer := c.detachNormalTimerLocked()
1077		c.pending, c.pendingReset, c.pendingUrgent, c.ackDeferred, c.deadlineDue, c.coalesced, c.armed = false, false, false, false, false, 0, false	1198		c.pending, c.pendingReset, c.pendingUrgent, c.ackDeferred, c.deadlineDue, c.coalesced, c.armed = false, false, false, false, false, 0, false
			1199	+	observer, queueCorrelation, queueMarked := c.opts.observer, c.queueCorrelation, c.queueMarked
			1200	+	ackBlocked := c.ackBlocked
			1201	+	c.queueMarked, c.ackBlocked = false, nil
1078		wake := c.opts.wake	1202		wake := c.opts.wake
1079		c.mu.Unlock()	1203		c.mu.Unlock()
1080		stopTimer(timer)	1204		stopTimer(timer)
			1205	+	if queueMarked && observer != nil {
			1206	+	observer.ObserveRuntime(ports.NewRuntimeMarkWithCorrelation("daemon", queueCorrelation, ports.RuntimeQueueDequeued, 0, true))
			1207	+	}
			1208	+	if ackBlocked != nil {
			1209	+	ackBlocked.finish(true)
			1210	+	}
1081		if wake != nil && !headlessPreviewOnly {	1211		if wake != nil && !headlessPreviewOnly {
1082		wake(w)	1212		wake(w)
1083		}	1213		}

M internal/usecase/daemon/render_pipeline.go +16 -1

 // emitFrame is the sole side-effecting half of the pipeline. The caller holds
 // sendMu for the complete capture/compose/emit transaction.
-func (d *Daemon) emitFrame(sess *session, ac *attachedClient, state *capturedRenderState, composed composedRenderFrame) bool {
+func (d *Daemon) emitFrame(sess *session, ac *attachedClient, state *capturedRenderState, composed composedRenderFrame, batches ...*runtimeMarkBatch) bool {
+	var ownedMarks runtimeMarkBatch
+	var marks *runtimeMarkBatch
+	if len(batches) != 0 {
+		marks = batches[0]
+	} else {
+		ownedMarks = d.newRuntimeMarkBatch()
+		marks = &ownedMarks
+		// Direct test callers retain the same guarantee as paint: observer I/O
+		// follows emitFrame's release of attachment ownership.
+		defer marks.flush()
+	}
 	if sess == nil || ac == nil || state == nil {
 		if ac != nil {
 			ac.sendMu.Unlock()
 		ac.sendMu.Unlock()
 		return false
 	}
+	endDiff := marks.span(ports.RuntimeDiffStart, ports.RuntimeDiffEnd, 0)
 	prepared, err := ac.output.prepare(composed.frame, composed.damage, composed.reset)
+	endDiff(0, err == nil)
 	if err != nil {
 		ac.sess.mu.Unlock()
 		ac.sendMu.Unlock()
 	var sendErr error
 	if len(data) > 0 {
 		sendTr = ac.transport()
+		endEmit := marks.span(ports.RuntimeEmitStart, ports.RuntimeEmitEnd, uint64(len(data)))
 		if sendTr == nil {
 			sendErr = errors.New("client transport is nil")
 		} else {
 			}
 			sendErr = prepared.send(data, ac.echoAck.Load(), send)
 		}
+		endEmit(uint64(len(data)), sendErr == nil)
 	}
 	if sendErr == nil {
 		// Publish only after output preparation and transport emission both

M internal/usecase/daemon/transactional_resize.go +5 -1

 import (
 	"github.com/bnema/vev/internal/domain"
+	"github.com/bnema/vev/internal/ports"
 	"github.com/bnema/vev/internal/usecase/layout"
 )
 		ac.sendMu.Unlock()
 	}
 	markSnapshotDirty(sess)
+	d.observeRuntime(ports.RuntimeResizeCommitted, 0, true)
 }
 func (d *Daemon) runResizeTransaction(sess *session, ac *attachedClient, epoch uint64) bool {
 // attached requests, and coordinator schedule acceptance for async requests.
 // Headless requests have no reset invalidation and report geometry completion.
 func (d *Daemon) requestTransactionalResize(sess *session, ac *attachedClient, size domain.Size, immediate bool) bool {
-	if sess == nil || !size.Valid() {
+	valid := sess != nil && size.Valid()
+	d.observeRuntime(ports.RuntimeResizeRequested, 0, valid)
+	if !valid {
 		return false
 	}
 	if ac == nil {

A internal/usecase/daemon/transport_observability_test.go +388 -0

@@ -0,0 +1,388 @@
	1	+	package daemon
	2	+
	3	+	import (
	4	+	"runtime"
	5	+	"sync"
	6	+	"sync/atomic"
	7	+	"testing"
	8	+	"time"
	9	+
	10	+	"github.com/bnema/vev/internal/domain"
	11	+	"github.com/bnema/vev/internal/ports"
	12	+	)
	13	+
	14	+	// This acceptance boundary exercises the attached resize, render, and ACK
	15	+	// path rather than merely listing the marks the implementation should emit.
	16	+	func TestTransportObservabilityDaemonBoundaries(t *testing.T) {
	17	+	observer := &daemonRuntimeObserver{}
	18	+	d, sess, ac, sends := newManualSessionWithPTYs(t, &quietPTY{})
	19	+	d.runtimeObserver = observer
	20	+	// One in-flight state makes the second public resize take the ACK-blocked
	21	+	// path. The real coordinator owns both renders and the ACK wake.
	22	+	ac.output = newOutputStateStream(1)
	23	+	rc := d.attachCoordinator(sess, nil, ac, true)
	24	+
	25	+	if !d.resizeForFirstPaint(sess, ac, domain.Size{Cols: 90, Rows: 25}) {
	26	+	t.Fatal("first resize was not accepted")
	27	+	}
	28	+	first := <-sends
	29	+	if first.Type != ports.MsgOutput {
	30	+	t.Fatalf("first frame = %v, want output", first.Type)
	31	+	}
	32	+	output, err := ports.UnmarshalOutput(first.Payload)
	33	+	if err != nil {
	34	+	t.Fatalf("decode first output: %v", err)
	35	+	}
	36	+	if !d.resizeForFirstPaint(sess, ac, domain.Size{Cols: 100, Rows: 26}) {
	37	+	t.Fatal("second resize was not accepted")
	38	+	}
	39	+	ac.ackOutputState(output.NewStateNum)
	40	+	rc.notifyAck()
	41	+	second := <-sends
	42	+	if second.Type != ports.MsgOutput {
	43	+	t.Fatalf("second frame = %v, want output", second.Type)
	44	+	}
	45	+
	46	+	want := []ports.RuntimeMarkKind{
	47	+	ports.RuntimeResizeRequested, ports.RuntimeResizeCommitted,
	48	+	ports.RuntimeQueueEnqueued, ports.RuntimeQueueDequeued,
	49	+	ports.RuntimeCaptureStart, ports.RuntimeCaptureEnd,
	50	+	ports.RuntimeComposeStart, ports.RuntimeComposeEnd,
	51	+	ports.RuntimeDiffStart, ports.RuntimeDiffEnd,
	52	+	ports.RuntimeEmitStart, ports.RuntimeEmitEnd,
	53	+	ports.RuntimeResizeRequested, ports.RuntimeResizeCommitted,
	54	+	ports.RuntimeQueueEnqueued, ports.RuntimeACKBlockedStart,
	55	+	ports.RuntimeACKBlockedEnd, ports.RuntimeQueueDequeued,
	56	+	ports.RuntimeCaptureStart, ports.RuntimeCaptureEnd,
	57	+	ports.RuntimeComposeStart, ports.RuntimeComposeEnd,
	58	+	ports.RuntimeDiffStart, ports.RuntimeDiffEnd,
	59	+	ports.RuntimeEmitStart, ports.RuntimeEmitEnd,
	60	+	}
	61	+	if len(observer.marks) != len(want) {
	62	+	t.Fatalf("runtime marks = %#v, want kinds %#v", observer.marks, want)
	63	+	}
	64	+	for i, kind := range want {
	65	+	if observer.marks[i].Kind != kind {
	66	+	t.Fatalf("runtime mark %d = %q, want %q; all=%#v", i, observer.marks[i].Kind, kind, observer.marks)
	67	+	}
	68	+	}
	69	+	}
	70	+
	71	+	type daemonRuntimeObserver struct{ marks []ports.RuntimeMark }
	72	+
	73	+	func (o *daemonRuntimeObserver) ObserveRuntime(mark ports.RuntimeMark) {
	74	+	o.marks = append(o.marks, mark)
	75	+	}
	76	+
	77	+	type blockingDaemonRuntimeObserver struct {
	78	+	mu sync.Mutex
	79	+	entered chan struct{}
	80	+	release chan struct{}
	81	+	blocked bool
	82	+	}
	83	+
	84	+	func newBlockingDaemonRuntimeObserver() *blockingDaemonRuntimeObserver {
	85	+	return &blockingDaemonRuntimeObserver{entered: make(chan struct{}), release: make(chan struct{})}
	86	+	}
	87	+
	88	+	func (o *blockingDaemonRuntimeObserver) ObserveRuntime(ports.RuntimeMark) {
	89	+	o.mu.Lock()
	90	+	defer o.mu.Unlock()
	91	+	if !o.blocked {
	92	+	o.blocked = true
	93	+	close(o.entered)
	94	+	<-o.release
	95	+	}
	96	+	}
	97	+
	98	+	func awaitDaemonObserver(t *testing.T, ch <-chan struct{}, what string) {
	99	+	t.Helper()
	100	+	select {
	101	+	case <-ch:
	102	+	case <-time.After(time.Second):
	103	+	t.Fatalf("timed out waiting for %s", what)
	104	+	}
	105	+	}
	106	+
	107	+	// A runtime observer may synchronously write JSONL. Its I/O must run only
	108	+	// after the render transaction has released attachment, session, and pane
	109	+	// ownership, so a blocked trace cannot stall live output processing.
	110	+	func TestTransportObservabilityBlockedRenderObserverReleasesArchitectureLocks(t *testing.T) {
	111	+	d, sess, ac, sends := newManualSessionWithPTYs(t, nil)
	112	+	observer := newBlockingDaemonRuntimeObserver()
	113	+	d.runtimeObserver = observer
	114	+
	115	+	paintDone := make(chan struct{})
	116	+	go func() {
	117	+	d.paint(sess, ac, true)
	118	+	close(paintDone)
	119	+	}()
	120	+	awaitDaemonObserver(t, observer.entered, "blocked render observer")
	121	+	frame := <-sends // The frame was emitted before the observer began its I/O.
	122	+	output, err := ports.UnmarshalOutput(frame.Payload)
	123	+	if err != nil {
	124	+	t.Fatalf("decode emitted output: %v", err)
	125	+	}
	126	+	ac.ackOutputState(output.NewStateNum)
	127	+
	128	+	pane := sess.activeTab().focusedPane()
	129	+	for _, lock := range []struct {
	130	+	name string
	131	+	fn func()
	132	+	}{
	133	+	{"attachment", func() { ac.sendMu.Lock(); runtime.Gosched(); ac.sendMu.Unlock() }},
	134	+	{"session", func() { sess.mu.Lock(); runtime.Gosched(); sess.mu.Unlock() }},
	135	+	{"pane", func() { pane.mu.Lock(); runtime.Gosched(); pane.mu.Unlock() }},
	136	+	} {
	137	+	done := make(chan struct{})
	138	+	go func(fn func()) { fn(); close(done) }(lock.fn)
	139	+	awaitDaemonObserver(t, done, lock.name+" lock")
	140	+	}
	141	+
	142	+	livePTY := make(chan struct{})
	143	+	go func() {
	144	+	d.processPTYData(sess, sess.activeTab(), pane, []byte("live"), false)
	145	+	close(livePTY)
	146	+	}()
	147	+	awaitDaemonObserver(t, livePTY, "live PTY processing")
	148	+
	149	+	// A second render can complete its actual pipeline while the first trace
	150	+	// write is blocked; it is not serialized behind the first attachment lock.
	151	+	secondPaint := make(chan struct{})
	152	+	go func() {
	153	+	d.paint(sess, ac, false)
	154	+	close(secondPaint)
	155	+	}()
	156	+	<-sends // The second frame is live even though serialized observer I/O waits.
	157	+
	158	+	close(observer.release)
	159	+	awaitDaemonObserver(t, paintDone, "first render completion")
	160	+	awaitDaemonObserver(t, secondPaint, "unrelated render completion")
	161	+	}
	162	+
	163	+	// ACK-blocked start is captured while coordinator state changes, then emitted
	164	+	// after c.mu is unlocked. This closes the remaining coordinator-observer path.
	165	+	func TestTransportObservabilityACKBlockedSpansEndExactlyOnce(t *testing.T) {
	166	+	cases := []struct {
	167	+	name string
	168	+	finish func(renderCoordinator, bool)
	169	+	}{
	170	+	{
	171	+	name: "notify ACK",
	172	+	finish: func(rc renderCoordinator, ready bool) {
	173	+	*ready = true
	174	+	rc.notifyAck()
	175	+	},
	176	+	},
	177	+	{
	178	+	name: "successful consume",
	179	+	finish: func(rc renderCoordinator, ready bool) {
	180	+	*ready = true
	181	+	rc.fire(rc.generation, false, false)
	182	+	},
	183	+	},
	184	+	{
	185	+	name: "teardown",
	186	+	finish: func(rc renderCoordinator, ready bool) {
	187	+	rc.noteSessionTeardown()
	188	+	},
	189	+	},
	190	+	}
	191	+	for _, tt := range cases {
	192	+	t.Run(tt.name, func(t *testing.T) {
	193	+	observer := &daemonRuntimeObserver{}
	194	+	ready := false
	195	+	rc := newRenderCoordinator(renderCoordinatorOptions{
	196	+	observer: observer,
	197	+	ackReady: func() bool { return ready },
	198	+	})
	199	+	rc.attach(&attachedClient{})
	200	+	rc.invalidate(renderInvalidation{class: invalidateOutput})
	201	+	rc.fire(rc.generation, false, true)
	202	+	tt.finish(rc, &ready)
	203	+	// A late notification after consuming or tearing down must not emit a
	204	+	// second end for the already-cleared blocked interval.
	205	+	rc.notifyAck()
	206	+
	207	+	var starts, ends []ports.RuntimeMark
	208	+	for _, mark := range observer.marks {
	209	+	switch mark.Kind {
	210	+	case ports.RuntimeACKBlockedStart:
	211	+	starts = append(starts, mark)
	212	+	case ports.RuntimeACKBlockedEnd:
	213	+	ends = append(ends, mark)
	214	+	}
	215	+	}
	216	+	if len(starts) != 1 \|\| len(ends) != 1 {
	217	+	t.Fatalf("ACK-blocked span marks: starts=%+v ends=%+v all=%+v", starts, ends, observer.marks)
	218	+	}
	219	+	if starts[0].Sequence != ends[0].Sequence \|\| starts[0].RequestID != ends[0].RequestID \|\| starts[0].Epoch != ends[0].Epoch {
	220	+	t.Fatalf("ACK-blocked span correlation changed: start=%+v end=%+v", starts[0], ends[0])
	221	+	}
	222	+	})
	223	+	}
	224	+	}
	225	+
	226	+	// A raw observer supplied through daemon configuration is serialized by the
	227	+	// daemon. Once Start has been published, a blocked End must not delay the ACK
	228	+	// that consumes deferred work and delivers its wake.
	229	+	func TestTransportObservabilityBlockedACKEndDoesNotDelayNotifyAck(t *testing.T) {
	230	+	observer := &blockingACKEndObserver{
	231	+	startObserved: make(chan struct{}),
	232	+	endEntered: make(chan struct{}),
	233	+	releaseEnd: make(chan struct{}),
	234	+	}
	235	+	d := New(nil, nil, nil, WithRuntimeObserver(observer))
	236	+	wakes := make(chan renderWake, 1)
	237	+	var ready atomic.Bool
	238	+	rc := newRenderCoordinator(renderCoordinatorOptions{
	239	+	observer: d.runtimeObserver,
	240	+	ackReady: ready.Load,
	241	+	wake: func(w renderWake) { wakes <- w },
	242	+	})
	243	+	rc.attach(&attachedClient{})
	244	+	rc.invalidate(renderInvalidation{class: invalidateOutput})
	245	+
	246	+	// Enter the ACK-blocked state and wait until Start has reached the injected
	247	+	// observer. End is the only deliberately blocked call in this test.
	248	+	rc.fire(rc.generation, false, true)
	249	+	awaitDaemonObserver(t, observer.startObserved, "ACK-blocked start observer")
	250	+
	251	+	ready.Store(true)
	252	+	ackDone := make(chan struct{})
	253	+	go func() {
	254	+	rc.notifyAck()
	255	+	close(ackDone)
	256	+	}()
	257	+	awaitDaemonObserver(t, observer.endEntered, "blocked ACK-blocked end observer")
	258	+	awaitDaemonObserver(t, ackDone, "nonblocking ACK progress")
	259	+	awaitWake(t, wakes)
	260	+	rc.mu.Lock()
	261	+	pending := rc.pending
	262	+	rc.mu.Unlock()
	263	+	if pending {
	264	+	t.Fatal("notifyAck left consumed ACK-deferred work pending")
	265	+	}
	266	+
	267	+	close(observer.releaseEnd)
	268	+	d.closeRuntimeObserver()
	269	+	}
	270	+
	271	+	type blockingACKEndObserver struct {
	272	+	startObserved chan struct{}
	273	+	endEntered chan struct{}
	274	+	releaseEnd chan struct{}
	275	+	}
	276	+
	277	+	func (o *blockingACKEndObserver) ObserveRuntime(mark ports.RuntimeMark) {
	278	+	switch mark.Kind {
	279	+	case ports.RuntimeACKBlockedStart:
	280	+	close(o.startObserved)
	281	+	case ports.RuntimeACKBlockedEnd:
	282	+	close(o.endEntered)
	283	+	<-o.releaseEnd
	284	+	}
	285	+	}
	286	+
	287	+	func TestTransportObservabilityBlockedCoordinatorObserverReleasesLock(t *testing.T) {
	288	+	observer := newBlockingDaemonRuntimeObserver()
	289	+	rc := newRenderCoordinator(renderCoordinatorOptions{
	290	+	observer: observer,
	291	+	ackReady: func() bool { return false },
	292	+	})
	293	+	rc.attach(&attachedClient{})
	294	+	rc.mu.Lock()
	295	+	rc.pending = true
	296	+	gen := rc.generation
	297	+	rc.mu.Unlock()
	298	+
	299	+	fired := make(chan struct{})
	300	+	go func() {
	301	+	rc.fire(gen, false, true)
	302	+	close(fired)
	303	+	}()
	304	+	awaitDaemonObserver(t, observer.entered, "blocked ACK observer")
	305	+
	306	+	acquired := make(chan struct{})
	307	+	go func() {
	308	+	rc.mu.Lock()
	309	+	runtime.Gosched()
	310	+	rc.mu.Unlock()
	311	+	close(acquired)
	312	+	}()
	313	+	awaitDaemonObserver(t, acquired, "coordinator lock")
	314	+
	315	+	close(observer.release)
	316	+	awaitDaemonObserver(t, fired, "ACK fire completion")
	317	+	}
	318	+
	319	+	type ackSpanInterleavingObserver struct {
	320	+	mu sync.Mutex
	321	+	marks []ports.RuntimeMark
	322	+	startEntered chan struct{}
	323	+	releaseStart chan struct{}
	324	+	}
	325	+
	326	+	func newACKSpanInterleavingObserver() *ackSpanInterleavingObserver {
	327	+	return &ackSpanInterleavingObserver{startEntered: make(chan struct{}), releaseStart: make(chan struct{})}
	328	+	}
	329	+
	330	+	func (o *ackSpanInterleavingObserver) ObserveRuntime(mark ports.RuntimeMark) {
	331	+	if mark.Kind == ports.RuntimeACKBlockedStart {
	332	+	close(o.startEntered)
	333	+	<-o.releaseStart
	334	+	}
	335	+	o.mu.Lock()
	336	+	o.marks = append(o.marks, mark)
	337	+	o.mu.Unlock()
	338	+	}
	339	+
	340	+	func (o *ackSpanInterleavingObserver) snapshot() []ports.RuntimeMark {
	341	+	o.mu.Lock()
	342	+	defer o.mu.Unlock()
	343	+	return append([]ports.RuntimeMark(nil), o.marks...)
	344	+	}
	345	+
	346	+	// An ACK can consume deferred work while the start mark is still waiting on
	347	+	// observer I/O. The End must wait for that Start publication, but the ACK
	348	+	// itself must not wait for either observer call.
	349	+	func TestTransportObservabilityACKBlockedStartPublishesBeforeEnd(t *testing.T) {
	350	+	observer := newACKSpanInterleavingObserver()
	351	+	ready := false
	352	+	rc := newRenderCoordinator(renderCoordinatorOptions{
	353	+	observer: observer,
	354	+	ackReady: func() bool { return ready },
	355	+	})
	356	+	rc.mu.Lock()
	357	+	rc.pending = true
	358	+	gen := rc.generation
	359	+	rc.mu.Unlock()
	360	+
	361	+	fireDone := make(chan struct{})
	362	+	go func() {
	363	+	rc.fire(gen, false, true)
	364	+	close(fireDone)
	365	+	}()
	366	+	awaitDaemonObserver(t, observer.startEntered, "ACK-blocked start observer")
	367	+
	368	+	ready = true
	369	+	ackDone := make(chan struct{})
	370	+	go func() {
	371	+	rc.notifyAck()
	372	+	close(ackDone)
	373	+	}()
	374	+	awaitDaemonObserver(t, ackDone, "nonblocking ACK progress")
	375	+	if marks := observer.snapshot(); len(marks) != 0 {
	376	+	t.Fatalf("ACK-blocked end published before its blocked start: %#v", marks)
	377	+	}
	378	+
	379	+	close(observer.releaseStart)
	380	+	awaitDaemonObserver(t, fireDone, "ACK-blocked start completion")
	381	+	marks := observer.snapshot()
	382	+	if len(marks) != 2 \|\| marks[0].Kind != ports.RuntimeACKBlockedStart \|\| marks[1].Kind != ports.RuntimeACKBlockedEnd {
	383	+	t.Fatalf("ACK-blocked marks = %#v, want one ordered start/end pair", marks)
	384	+	}
	385	+	if marks[0].Sequence != marks[1].Sequence \|\| marks[0].RequestID != marks[1].RequestID \|\| marks[0].Epoch != marks[1].Epoch {
	386	+	t.Fatalf("ACK-blocked span correlation changed: start=%+v end=%+v", marks[0], marks[1])
	387	+	}
	388	+	}

A testdata/perf/README.md +69 -0

@@ -0,0 +1,69 @@
	1	+	# S5a transport-observability evidence
	2	+
	3	+	`manifest.json` is the complete, measurement-only public-CLI matrix: four
	4	+	120x40 topologies (`1x4`, `4x1`, `4x4`, `8x1`), 10,000 history rows per pane,
	5	+	nine workloads, and seven transports (local, SSH stdio, UDP baseline, UDP 25 ms,
	6	+	UDP 100 ms, and UDP with 0% or 1% loss). It contains 252 scenario combinations;
	7	+	there are no omitted combinations. A genuinely unavailable combination must
	8	+	remain listed with `inapplicable_reason` and no launched roles.
	9	+
	10	+	Build and run the harness with at least a 10-second warmup, 30-second measured
	11	+	interval, and 10 independent repetitions:
	12	+
	13	+	```sh
	14	+	go build -o /tmp/vev-perf ./
	15	+	rm -rf testdata/perf/results
	16	+	go run ./cmd/vev-perf-harness --vev-bin /tmp/vev-perf \
	17	+	--manifest testdata/perf/manifest.json --out testdata/perf/results \
	18	+	--warmup 10s --duration 30s --repetitions 10
	19	+	```
	20	+
	21	+	Each run writes `manifest.json`, `raw-harness.jsonl`, one JSONL trace per
	22	+	launched process, `runs.json`, and `summary.json`. The process manifest records
	23	+	process ID, role, scenario/run, clock domain, exact exclusive trace path, and
	24	+	identity. Every launched role receives `VEV_PERF_TRACE`, `VEV_PERF_PROCESS_ID`,
	25	+	`VEV_PERF_SCENARIO`, and `VEV_PERF_RUN`; the first is exclusive to that process
	26	+	and the remaining values correlate it to the manifest. The harness owns only the end-to-end boundaries: input injection to
	27	+	terminal bytes successfully flushed. It never subtracts ticks from daemon,
	28	+	client, or adapter processes.
	29	+
	30	+	A process trace is JSONL with exactly these keys and types:
	31	+
	32	+	```json
	33	+	{"schema":1,"process_id":"p","component":"daemon","scenario":"s","run":1,"sequence":1,"request_id":1,"epoch":1,"kind":"diff_start","tick":0,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	34	+	```
	35	+
	36	+	`tick` is owned and stamped by that process's injected monotonic clock. Producers
	37	+	must leave it zero; one observer/clock is shared by components in a process.
	38	+	Ticks from different `process_id` or clock domains are never ordered or
	39	+	subtracted. Correlation uses scenario, run, sequence, request ID, and epoch.
	40	+
	41	+	Named process-local spans are: capture, compose, diff (`diff_start/end`), queue
	42	+	wait (`queue_enqueued/dequeued`), ACK-blocked interval, emit, adapter send,
	43	+	and adapter receive. Adapter spans are reported by component/transport; bytes,
	44	+	fragments, retransmits, pending, and ACK RTT remain diagnostics. Reports retain
	45	+	raw records and counts plus p50/p95/p99 (and max where applicable). End-to-end
	46	+	reports are harness-clock distributions only.
	47	+
	48	+	The harness injects at most one measured event per second and requires at least
	49	+	10 complete in-interval event pairs. Both the input-injection and successful
	50	+	terminal-flush boundaries must be within the measured interval; warmup and
	51	+	straddling pairs are excluded from percentile samples (but retained as raw
	52	+	evidence). Reject a result for missing or unmatched boundaries, missing flush,
	53	+	invalid correlation, nonmonotonic same-process sequence, negative same-domain
	54	+	duration, fewer than 10 complete in-interval pairs, invalid denominators,
	55	+	cross-process tick math, less than 30 seconds measured, or fewer than 10
	56	+	repetitions. Warmup is excluded from measured samples. Raw traces are evidence, not a transport-policy or
	57	+	optimization gate; wire bytes, protocol version, rendering policy, pacing,
	58	+	compression, and loss behavior are unchanged.
	59	+
	60	+	## Bounded manifest and harness checks
	61	+
	62	+	Before a measurement, validate the manifest shape and run the harness tests without launching the full matrix:
	63	+
	64	+	```sh
	65	+	jq -e '(.topologies\|length)==4 and (.workloads\|length)==9 and (.transports\|length)==7 and (.scenarios\|length)==252' testdata/perf/manifest.json
	66	+	go test ./cmd/vev-perf-harness
	67	+	```
	68	+
	69	+	The full command above is the measurement procedure; its 10-second warmup, 30-second measured interval, and 10 repetitions are the minimum accepted settings. For a bounded local evidence run, add `--scenario 1x4-idle-local`; the harness validates the complete canonical manifest before it applies that selection, so a single-scenario run cannot hide missing matrix coverage.

A testdata/perf/manifest.json +2808 -0 large diff (2808 lines changed): collapsed

load diff