1//go:build linux
2
3package main
4
5import (
6 "bytes"
7 "encoding/json"
8 "errors"
9 "fmt"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "strings"
14 "testing"
15 "time"
16
17 "github.com/bnema/vev/pkg/safedir"
18)
19
20func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
21 if testing.Short() {
22 t.Skip("public CLI repetition lifecycle")
23 }
24 root, err := filepath.Abs(filepath.Join("..", ".."))
25 if err != nil {
26 t.Fatal(err)
27 }
28 bin := filepath.Join(t.TempDir(), "vev")
29 build := exec.Command("go", "build", "-o", bin, "./")
30 build.Dir = root
31 if output, err := build.CombinedOutput(); err != nil {
32 t.Fatalf("build public CLI: %v\n%s", err, output)
33 }
34 m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
35 if err != nil {
36 t.Fatal(err)
37 }
38 var local scenario
39 for _, candidate := range m.Scenarios {
40 if candidate.ID == "1x4-idle-local" {
41 local = candidate
42 break
43 }
44 }
45 if local.ID == "" {
46 t.Fatal("canonical local scenario missing")
47 }
48 out, err := os.MkdirTemp("", "vev-ob-")
49 if err != nil {
50 t.Fatal(err)
51 }
52 removeTestTree(t, out)
53 raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
54 if err != nil {
55 t.Fatal(err)
56 }
57 closeTestFile(t, raw)
58 h := defaultHarness()
59 h.clock = &fakeClock{}
60 h.launcher = &cliLauncher{bin: bin}
61 for run := 1; run <= 3; run++ {
62 result, err := h.runOne(options{vevBin: bin, out: out, duration: minimumDuration}, m, local, run, raw)
63 if err != nil {
64 t.Fatalf("run %d: %v", run, err)
65 }
66 if len(result.Processes) != 2 {
67 t.Fatalf("run %d process mappings = %+v", run, result.Processes)
68 }
69 runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
70 manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
71 if err != nil {
72 t.Fatalf("run %d manifest: %v", run, err)
73 }
74 var persisted runManifest
75 if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
76 t.Fatalf("run %d manifest JSON: %v", run, err)
77 }
78 if persisted.Run != run || len(persisted.Processes) != len(result.Processes) {
79 t.Fatalf("run %d manifest = %+v", run, persisted)
80 }
81 for _, process := range result.Processes {
82 if process.Role != "daemon" && process.Role != "client" {
83 t.Fatalf("run %d unexpected role %q", run, process.Role)
84 }
85 if filepath.Dir(process.TracePath) != runDir {
86 t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
87 }
88 trace, err := os.ReadFile(process.TracePath)
89 if err != nil {
90 t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
91 }
92 if len(bytes.TrimSpace(trace)) == 0 {
93 t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
94 }
95 for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
96 var record traceRecord
97 if err := json.Unmarshal(line, &record); err != nil {
98 t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
99 }
100 if record.ProcessID != process.ProcessID || record.Scenario != local.ID || record.Run != uint64(run) {
101 t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
102 }
103 }
104 }
105 for _, socket := range []string{
106 filepath.Join(runDir, "runtime", "vev", "broker", "broker.sock"),
107 filepath.Join(runDir, "runtime", "vev", "daemonmux.sock"),
108 } {
109 if _, err := os.Stat(socket); !errors.Is(err, os.ErrNotExist) {
110 t.Fatalf("run %d left production socket %q for the next repetition: %v", run, socket, err)
111 }
112 }
113 }
114}
115
116func TestHarnessResolvesCanonicalRelativePathsBeforeRoleCWDChanges(t *testing.T) {
117 if testing.Short() {
118 t.Skip("public CLI relative-path lifecycle")
119 }
120 root, err := filepath.Abs(filepath.Join("..", ".."))
121 if err != nil {
122 t.Fatal(err)
123 }
124 // Keep the real Unix-socket path below its platform limit while testing the
125 // exact documented relative output hierarchy.
126 workspace, err := os.MkdirTemp("", "vev-rel-")
127 if err != nil {
128 t.Fatal(err)
129 }
130 removeTestTree(t, workspace)
131 bin := filepath.Join(workspace, "vev")
132 build := exec.Command("go", "build", "-o", bin, "./")
133 build.Dir = root
134 if output, err := build.CombinedOutput(); err != nil {
135 t.Fatalf("build public CLI: %v\n%s", err, output)
136 }
137 manifestPath := filepath.Join(workspace, "testdata", "perf", "manifest.json")
138 if err := safedir.EnsurePrivate(filepath.Dir(manifestPath)); err != nil {
139 t.Fatal(err)
140 }
141 contents, err := os.ReadFile(filepath.Join(root, "testdata", "perf", "manifest.json"))
142 if err != nil {
143 t.Fatal(err)
144 }
145 if err := os.WriteFile(manifestPath, contents, 0o600); err != nil {
146 t.Fatal(err)
147 }
148 originalWD, err := os.Getwd()
149 if err != nil {
150 t.Fatal(err)
151 }
152 if err := os.Chdir(workspace); err != nil {
153 t.Fatal(err)
154 }
155 t.Cleanup(func() {
156 if err := os.Chdir(originalWD); err != nil {
157 t.Errorf("restore cwd: %v", err)
158 }
159 })
160
161 // These are the documented relative locations. cliLauncher changes every
162 // role to its run directory, so a failure to resolve them before launch
163 // would make VEV_PERF_TRACE point at a nonexistent nested directory.
164 h := defaultHarness()
165 h.clock = &fakeClock{}
166 if err := run([]string{
167 "--vev-bin", "./vev",
168 "--manifest", "testdata/perf/manifest.json",
169 "--out", "testdata/perf/results",
170 "--scenario", "1x4-idle-local",
171 "--warmup", "0s",
172 "--duration", "30s",
173 "--repetitions", "10",
174 }, h); err != nil {
175 t.Fatal(err)
176 }
177
178 out := filepath.Join(workspace, "testdata", "perf", "results")
179 for run := 1; run <= 10; run++ {
180 runDir := filepath.Join(out, fmt.Sprintf("1x4-idle-local-run-%03d", run))
181 manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
182 if err != nil {
183 t.Fatalf("run %d manifest: %v", run, err)
184 }
185 var persisted runManifest
186 if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
187 t.Fatalf("run %d manifest JSON: %v", run, err)
188 }
189 if persisted.Run != run || len(persisted.Processes) != 2 {
190 t.Fatalf("run %d manifest=%+v", run, persisted)
191 }
192 for _, process := range persisted.Processes {
193 if !filepath.IsAbs(process.TracePath) || filepath.Dir(process.TracePath) != runDir {
194 t.Fatalf("run %d %s trace path=%q, want absolute path in %q", run, process.Role, process.TracePath, runDir)
195 }
196 trace, err := os.ReadFile(process.TracePath)
197 if err != nil || len(bytes.TrimSpace(trace)) == 0 {
198 t.Fatalf("run %d %s trace=%q err=%v", run, process.Role, process.TracePath, err)
199 }
200 }
201 }
202}
203
204func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
205 out := t.TempDir()
206 raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
207 if err != nil {
208 t.Fatal(err)
209 }
210 closeTestFile(t, raw)
211 s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
212 stale := filepath.Join(out, "isolated-run-001")
213 if err := os.Mkdir(stale, 0o700); err != nil {
214 t.Fatal(err)
215 }
216 if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
217 t.Fatal(err)
218 }
219 launcher := &fakeLauncher{}
220 h := defaultHarness()
221 h.clock, h.launcher = &fakeClock{}, launcher
222 _, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
223 if err == nil || !strings.Contains(err.Error(), "create isolated run directory") {
224 t.Fatalf("existing run directory error = %v", err)
225 }
226 if len(launcher.mappings) != 0 {
227 t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
228 }
229 if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil || string(contents) != "stale" {
230 t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
231 }
232}
233
234func TestHarnessDoesNotInheritNestedSession(t *testing.T) {
235 env := withoutEnv([]string{"VEV=session=old", "PATH=/bin", "VEV_PERF_TRACE=old"}, "VEV")
236 if !equalStrings(env, []string{"PATH=/bin", "VEV_PERF_TRACE=old"}) {
237 t.Fatalf("environment=%q", env)
238 }
239}
240
241func TestHarnessRoutesEveryRemoteFixtureThroughItsDeclaredPeer(t *testing.T) {
242 cases := []struct {
243 name string
244 transport transport
245 peer string
246 }{
247 {"ssh", transport{ID: "ssh_stdio", Kind: "ssh_stdio"}, "ssh_stdio_peer"},
248 }
249 for _, tc := range cases {
250 t.Run(tc.name, func(t *testing.T) {
251 s := scenario{ID: "route", Transport: tc.transport.ID}
252 client := routeRoleArgs(s, processMapping{Role: "client", Run: 2}, tc.transport)
253 peer := routeRoleArgs(s, processMapping{Role: tc.peer, Run: 2}, tc.transport)
254 if !equalStrings(client.Args, []string{"attach", "harness@127.0.0.1"}) {
255 t.Fatalf("client did not use ephemeral public remote attach: %q", client.Args)
256 }
257 if !equalStrings(peer.Args, []string{"_broker-mux-stdio", "--production"}) {
258 t.Fatalf("ssh peer command=%q", peer.Args)
259 }
260 })
261 }
262}
263
264func TestHarnessFakeRunnerRoutesClientToPeerAndCleansEveryRole(t *testing.T) {
265 for _, tc := range []struct {
266 name, transport, peer, peerCommand string
267 }{
268 {"ssh", "ssh_stdio", "ssh_stdio_peer", "_broker-mux-stdio"},
269 } {
270 t.Run(tc.name, func(t *testing.T) {
271 dir := t.TempDir()
272 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
273 if err != nil {
274 t.Fatal(err)
275 }
276 closeTestFile(t, raw)
277 l := &fakeLauncher{}
278 h := defaultHarness()
279 h.clock, h.launcher = &fakeClock{}, l
280 s := scenario{ID: "routed", Transport: tc.transport, Roles: []string{"daemon", "client", tc.peer}}
281 if _, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, s, 1, raw); err != nil {
282 t.Fatal(err)
283 }
284 if len(l.mappings) != 3 || l.mappings[0].Role != tc.peer || l.mappings[1].Role != "daemon" || l.mappings[2].Role != "client" {
285 t.Fatalf("dependency launch order=%+v", l.mappings)
286 }
287 if l.commands[0].Args[0] != tc.peerCommand || l.commands[2].Args[0] != "attach" {
288 t.Fatalf("commands do not connect client through declared peer: %+v", l.commands)
289 }
290 for i, p := range l.process {
291 if !p.closed {
292 t.Fatalf("role %s was not cleaned up", l.mappings[i].Role)
293 }
294 }
295 })
296 }
297}
298
299func TestWorkloadInputShapes(t *testing.T) {
300 tests := []struct {
301 workload string
302 contains []string
303 }{
304 {"active_output", []string{`printf '\033[10;20H1'`}},
305 {"all_output", []string{"120", "vev perf line"}},
306 {"inactive_output", []string{
307 `printf '\033[38;2;20;120;220mred'`,
308 `printf '\033[38;2;220;80;40mgreen'`,
309 `printf '\033[38;2;80;220;120ms'`,
310 }},
311 {"interactive_flood", []string{"printf 'vev perf output\\n'"}},
312 {"snapshot_output_resize", []string{`\033[2J`, "vev perf snapshot-output-resize"}},
313 {"attach_restore_tab_switch", []string{`"$VEV_PERF_BIN" cmd next-tab`, `"$VEV_PERF_BIN" cmd previous-tab`}},
314 }
315 for _, tt := range tests {
316 t.Run(tt.workload, func(t *testing.T) {
317 got := string(workloadInput(scenario{ID: "s", Workload: tt.workload}, 1, "measured-1"))
318 for _, want := range tt.contains {
319 if !strings.Contains(got, want) {
320 t.Errorf("workload input %q does not contain %q", got, want)
321 }
322 }
323 if !strings.Contains(got, "__VEV_HARNESS_s_r1_measured-1__") {
324 t.Errorf("workload input has no terminal marker: %q", got)
325 }
326 })
327 }
328
329 activeWarmup := string(workloadInput(scenario{ID: "s", Workload: "active_output"}, 1, "warmup"))
330 activeOne := string(workloadInput(scenario{ID: "s", Workload: "active_output"}, 1, "measured-1"))
331 activeTwo := string(workloadInput(scenario{ID: "s", Workload: "active_output"}, 1, "measured-2"))
332 for phase, got := range map[string]string{"warmup": activeWarmup, "measured-1": activeOne, "measured-2": activeTwo} {
333 marker := fmt.Sprintf("__VEV_HARNESS_s_r1_%s__", phase)
334 wantByte := map[string]byte{"warmup": 'X', "measured-1": '1', "measured-2": '0'}[phase]
335 want := fmt.Sprintf("printf '\\033[10;20H%c'; : 's'; printf '%s\\n'\n", wantByte, marker)
336 if got != want {
337 t.Errorf("active %s input = %q, want exactly one changing cell write %q", phase, got, want)
338 }
339 }
340 if activeWarmup == activeOne || activeOne == activeTwo {
341 t.Fatalf("active one-cell payload did not change by sequence: %q %q %q", activeWarmup, activeOne, activeTwo)
342 }
343 inactive := string(workloadInput(scenario{ID: "s", Workload: "inactive_output"}, 1, "measured-1"))
344 if count := strings.Count(inactive, `\033[38;2;`); count != 3 {
345 t.Fatalf("inactive output truecolor runs = %d, want 3: %q", count, inactive)
346 }
347
348 interactive := string(workloadInput(scenario{ID: "s", Workload: "interactive_flood"}, 1, "measured-1"))
349 if count := strings.Count(interactive, `printf 'vev perf output\n'`); count != 128 {
350 t.Fatalf("interactive flood printf commands = %d, want 128: %q", count, interactive)
351 }
352 if strings.Contains(interactive, "i=0") || strings.Contains(interactive, "while [") {
353 t.Fatalf("interactive flood uses shell-specific loop syntax: %q", interactive)
354 }
355
356 warmup := string(workloadInput(scenario{ID: "s", Workload: "attach_restore_tab_switch"}, 1, "warmup"))
357 if count := strings.Count(warmup, `"$VEV_PERF_BIN" cmd new-tab`); count != 1 {
358 t.Fatalf("attach warmup new-tab commands = %d, want 1: %q", count, warmup)
359 }
360 wantWarmup := `"$VEV_PERF_BIN" cmd new-tab && "$VEV_PERF_BIN" cmd previous-tab && printf 'vev perf attach-restore-tab-switch s\n' && printf '__VEV_HARNESS_s_r1_warmup__\n'` + "\n"
361 if warmup != wantWarmup {
362 t.Fatalf("attach warmup success marker is not gated by every command:\n got %q\nwant %q", warmup, wantWarmup)
363 }
364 measured := string(workloadInput(scenario{ID: "s", Workload: "attach_restore_tab_switch"}, 1, "measured-1"))
365 wantMeasured := `"$VEV_PERF_BIN" cmd next-tab && "$VEV_PERF_BIN" cmd previous-tab && printf 'vev perf attach-restore-tab-switch s\n' && printf '__VEV_HARNESS_s_r1_measured-1__\n'` + "\n"
366 if measured != wantMeasured {
367 t.Fatalf("attach measured success marker is not gated by every command:\n got %q\nwant %q", measured, wantMeasured)
368 }
369}
370
371func TestWorkloadInputResizeCommandsAreComplete(t *testing.T) {
372 for _, tt := range []struct {
373 workload string
374 want string
375 }{
376 {"resize_sweep", `printf 'vev perf resize-sweep s\n'; printf '__VEV_HARNESS_s_r1_measured-1__\n'` + "\n"},
377 {"snapshot_output_resize", `printf '\033[2J\033[Hvev perf snapshot-output-resize s\n'; printf '__VEV_HARNESS_s_r1_measured-1__\n'` + "\n"},
378 } {
379 t.Run(tt.workload, func(t *testing.T) {
380 got := string(workloadInput(scenario{ID: "s", Workload: tt.workload}, 1, "measured-1"))
381 if got != tt.want {
382 t.Fatalf("workload input = %q, want complete command %q", got, tt.want)
383 }
384 if strings.Contains(got, "%!") {
385 t.Fatalf("workload input contains fmt diagnostic: %q", got)
386 }
387 })
388 }
389}
390
391func TestWorkloadResize(t *testing.T) {
392 for _, tt := range []struct {
393 workload string
394 sequence uint64
395 wantCols uint16
396 wantOK bool
397 }{
398 {"active_output", 1, 0, false},
399 {"resize_sweep", 1, 119, true},
400 {"resize_sweep", 2, 120, true},
401 {"snapshot_output_resize", 1, 119, true},
402 } {
403 t.Run(fmt.Sprintf("%s-%d", tt.workload, tt.sequence), func(t *testing.T) {
404 cols, rows, ok := workloadResize(scenario{Workload: tt.workload}, tt.sequence)
405 if ok != tt.wantOK || cols != tt.wantCols {
406 t.Fatalf("workloadResize() = (%d, %d, %t), want cols=%d ok=%t", cols, rows, ok, tt.wantCols, tt.wantOK)
407 }
408 if ok && rows != 40 {
409 t.Fatalf("workloadResize() rows = %d, want 40", rows)
410 }
411 })
412 }
413}
414
415type resizeOrderingWriter struct {
416 bytes.Buffer
417 order *[]string
418}
419
420func (w *resizeOrderingWriter) Write(p []byte) (int, error) {
421 if bytes.Contains(p, []byte(`"sequence":2`)) {
422 switch {
423 case bytes.Contains(p, []byte(`"kind":"input_injected"`)):
424 *w.order = append(*w.order, "injected")
425 case bytes.Contains(p, []byte(`"kind":"terminal_flushed"`)):
426 *w.order = append(*w.order, "terminal_flushed")
427 }
428 }
429 return w.Buffer.Write(p)
430}
431
432type resizeOrderingProcess struct {
433 order *[]string
434 inputs [][]byte
435 resizes int
436 measures int
437}
438
439func (*resizeOrderingProcess) Warmup([]byte) error { return nil }
440
441func (p *resizeOrderingProcess) Resize(cols, rows uint16) error {
442 p.resizes++
443 if p.resizes == 1 {
444 *p.order = append(*p.order, fmt.Sprintf("resize:%dx%d", cols, rows))
445 }
446 return nil
447}
448
449func (p *resizeOrderingProcess) Measure(input []byte, injected, flushed func() error) error {
450 p.measures++
451 p.inputs = append(p.inputs, append([]byte(nil), input...))
452 if p.measures == 1 {
453 *p.order = append(*p.order, "workload_write")
454 }
455 if err := injected(); err != nil {
456 return err
457 }
458 if p.measures == 1 {
459 *p.order = append(*p.order, "workload_flush")
460 }
461 return flushed()
462}
463
464func (*resizeOrderingProcess) Close() error { return nil }
465
466type resizeOrderingLauncher struct{ client *resizeOrderingProcess }
467
468func (l resizeOrderingLauncher) Launch(m processMapping, _ roleCommand) (launchedProcess, error) {
469 if m.Role == "client" {
470 return l.client, nil
471 }
472 return &fakeProcess{}, nil
473}
474
475func TestResizeBoundaryPrecedesWorkloadWriteAndMarker(t *testing.T) {
476 dir := t.TempDir()
477 order := []string{}
478 client := &resizeOrderingProcess{order: &order}
479 h := defaultHarness()
480 h.clock, h.launcher = &fakeClock{}, resizeOrderingLauncher{client: client}
481 raw := &resizeOrderingWriter{order: &order}
482 _, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "resize", Workload: "resize_sweep", Roles: []string{"daemon", "client"}}, 1, raw)
483 if err != nil {
484 t.Fatal(err)
485 }
486 wantOrder := []string{"injected", "resize:120x40", "workload_write", "workload_flush", "terminal_flushed"}
487 if len(order) < len(wantOrder) {
488 t.Fatalf("resize event order = %q, want prefix %q", order, wantOrder)
489 }
490 if !equalStrings(order[:len(wantOrder)], wantOrder) {
491 t.Fatalf("first resize event order = %q, want %q", order[:len(wantOrder)], wantOrder)
492 }
493 first := string(client.inputs[0])
494 for _, want := range []string{`printf 'vev perf resize-sweep resize\n'`, `printf '__VEV_HARNESS_resize_r1_measured-2__\n'`} {
495 if !strings.Contains(first, want) {
496 t.Errorf("resize input has no observable workload or marker %q: %q", want, first)
497 }
498 }
499 if strings.Contains(first, "while [") || strings.Contains(first, "%!") {
500 t.Fatalf("resize input contains unsupported loop or fmt diagnostic: %q", first)
501 }
502}
503
504func TestHarnessAggregatesRawEventsSeparatelyFromRunDispersion(t *testing.T) {
505 first := append([]int64(nil), make([]int64, minimumInIntervalEventSamples+5)...)
506 for i := range first {
507 first[i] = 1
508 }
509 second := []int64{100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1_000, 1_000}
510 all, _, _, runP50s := aggregateEventResults([]runResult{
511 {EndToEnd: first, Event: distribution{P50: 1}},
512 {EndToEnd: second, Event: distribution{P50: 100}},
513 })
514 if got, dispersion := percentiles(all), percentiles(runP50s); got.Count != 30 || got.P99 != 1_000 || dispersion.Count != 2 || dispersion.P99 != 1 {
515 t.Fatalf("event aggregation=%+v run dispersion=%+v", got, dispersion)
516 }
517}
518
519func TestHarnessRawRecordsContainEveryMarkWhileSamplesRemainFiltered(t *testing.T) {
520 dir := t.TempDir()
521 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
522 if err != nil {
523 t.Fatal(err)
524 }
525 h := defaultHarness()
526 h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
527 result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "raw-complete", Roles: []string{"daemon", "client"}}, 1, raw)
528 if err != nil {
529 t.Fatal(err)
530 }
531 if err := raw.Close(); err != nil {
532 t.Fatal(err)
533 }
534 marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
535 if got, want := len(marks), 2*(result.Samples+1); got != want {
536 t.Fatalf("raw marks=%d, want warmup plus every measured boundary=%d", got, want)
537 }
538 for i, mark := range marks[:2] {
539 if mark.Sequence != 1 || !mark.Valid || []string{"input_injected", "terminal_flushed"}[i] != mark.Kind {
540 t.Fatalf("warmup mark %d=%+v", i, mark)
541 }
542 }
543 for sequence := uint64(2); sequence <= uint64(result.Samples+1); sequence++ {
544 if marks[2*(sequence-1)].Sequence != sequence || marks[2*(sequence-1)+1].Sequence != sequence {
545 t.Fatalf("raw marks did not retain both boundaries for sequence %d: %+v", sequence, marks)
546 }
547 }
548}
549
550func TestHarnessPersistsFailedFlushDiagnostics(t *testing.T) {
551 dir := t.TempDir()
552 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
553 if err != nil {
554 t.Fatal(err)
555 }
556 failure := errors.New("flush failed")
557 h := defaultHarness()
558 h.clock, h.launcher = &fakeClock{}, failedFlushLauncher{err: failure}
559 _, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "failed-flush", Roles: []string{"daemon", "client"}}, 1, raw)
560 if !errors.Is(err, failure) {
561 t.Fatalf("run error=%v, want failed flush", err)
562 }
563 if err := raw.Close(); err != nil {
564 t.Fatal(err)
565 }
566 marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
567 if got, want := len(marks), 4; got != want {
568 t.Fatalf("raw marks=%d, want warmup and failed measured pair=%d: %+v", got, want, marks)
569 }
570 failed := marks[len(marks)-1]
571 if failed.Sequence != 2 || failed.Kind != "terminal_flushed" || failed.Valid {
572 t.Fatalf("failed flush diagnostic=%+v", failed)
573 }
574}
575
576func TestHarnessSchedulesMeasuredEventsAcrossFullInterval(t *testing.T) {
577 dir := t.TempDir()
578 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
579 if err != nil {
580 t.Fatal(err)
581 }
582 closeTestFile(t, raw)
583 h := defaultHarness()
584 h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
585 result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "schedule", Roles: []string{"daemon", "client"}}, 1, raw)
586 if err != nil {
587 t.Fatal(err)
588 }
589 if result.Samples < minimumInIntervalEventSamples || result.Cadence.Count != result.Samples-1 || result.MaxGap <= 0 {
590 t.Fatalf("result did not include repeated measured events: %+v", result)
591 }
592 if err := raw.Close(); err != nil {
593 t.Fatal(err)
594 }
595 records, err := readJSONL(filepath.Join(dir, "raw.jsonl"))
596 if err != nil {
597 t.Fatal(err)
598 }
599 if want := result.Samples*2 + 2; len(records) != want {
600 t.Fatalf("raw records=%d, want %d warmup and measured boundaries", len(records), want)
601 }
602}
603
604func TestHarnessPreservesWorkloadAndCleanupErrors(t *testing.T) {
605 dir := t.TempDir()
606 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
607 if err != nil {
608 t.Fatal(err)
609 }
610 closeTestFile(t, raw)
611 primary := errors.New("primary workload failure")
612 cleanup := errors.New("daemon cleanup failure")
613 launcher := &fakeLauncher{
614 measureErr: map[string]error{"client": primary},
615 closeErr: map[string]error{"daemon": cleanup},
616 }
617 h := defaultHarness()
618 h.clock, h.launcher = &fakeClock{}, launcher
619 _, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "errors", Roles: []string{"daemon", "client"}}, 1, raw)
620 if !errors.Is(err, primary) || !errors.Is(err, cleanup) {
621 t.Fatalf("workload and cleanup errors were not both preserved: %v", err)
622 }
623 if got, want := err.Error(), primary.Error()+"\nclose daemon role: "+cleanup.Error(); got != want {
624 t.Fatalf("error ordering=%q want %q", got, want)
625 }
626 for i, p := range launcher.process {
627 if !p.closed {
628 t.Fatalf("role %s was not closed", launcher.mappings[i].Role)
629 }
630 }
631}
632
633func TestHarnessCleansRunDirectoryOnProcessFailure(t *testing.T) {
634 dir := t.TempDir()
635 raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
636 if err != nil {
637 t.Fatal(err)
638 }
639 closeTestFile(t, raw)
640 launcher := &fakeLauncher{}
641 h := defaultHarness()
642 h.clock = &fakeClock{}
643 // The client is the second role and fails after the daemon has started.
644 h.launcher = &failingLauncher{fakeLauncher: launcher}
645 o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
646 if _, err := h.runOne(o, manifest{}, scenario{ID: "cleanup", Roles: []string{"daemon", "client"}}, 1, raw); err == nil {
647 t.Fatal("failed process accepted")
648 }
649 if _, err := os.Stat(filepath.Join(dir, "cleanup-run-001")); !os.IsNotExist(err) {
650 t.Fatalf("failed run directory remains: %v", err)
651 }
652}