1//go:build linux
2
3package main
4
5import (
6 "bytes"
7 "encoding/json"
8 "errors"
9 "fmt"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "strings"
14 "testing"
15 "time"
16
17 "github.com/bnema/vev/pkg/safedir"
18)
19
20func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
21 if testing.Short() {
22 t.Skip("public CLI repetition lifecycle")
23 }
24 root, err := filepath.Abs(filepath.Join("..", ".."))
25 if err != nil {
26 t.Fatal(err)
27 }
28 bin := filepath.Join(t.TempDir(), "vev")
29 build := exec.Command("go", "build", "-o", bin, "./")
30 build.Dir = root
31 if output, err := build.CombinedOutput(); err != nil {
32 t.Fatalf("build public CLI: %v\n%s", err, output)
33 }
34 m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
35 if err != nil {
36 t.Fatal(err)
37 }
38 var local scenario
39 for _, candidate := range m.Scenarios {
40 if candidate.ID == "1x4-idle-local" {
41 local = candidate
42 break
43 }
44 }
45 if local.ID == "" {
46 t.Fatal("canonical local scenario missing")
47 }
48 out, err := os.MkdirTemp("", "vev-ob-")
49 if err != nil {
50 t.Fatal(err)
51 }
52 removeTestTree(t, out)
53 raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
54 if err != nil {
55 t.Fatal(err)
56 }
57 closeTestFile(t, raw)
58 h := defaultHarness()
59 h.clock = &fakeClock{}
60 h.launcher = &cliLauncher{bin: bin}
61 for run := 1; run <= 3; run++ {
62 result, err := h.runOne(options{vevBin: bin, out: out, duration: minimumDuration}, m, local, run, raw)
63 if err != nil {
64 t.Fatalf("run %d: %v", run, err)
65 }
66 if len(result.Processes) != 2 {
67 t.Fatalf("run %d process mappings = %+v", run, result.Processes)
68 }
69 runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
70 manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
71 if err != nil {
72 t.Fatalf("run %d manifest: %v", run, err)
73 }
74 var persisted runManifest
75 if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
76 t.Fatalf("run %d manifest JSON: %v", run, err)
77 }
78 if persisted.Run != run || len(persisted.Processes) != len(result.Processes) {
79 t.Fatalf("run %d manifest = %+v", run, persisted)
80 }
81 for _, process := range result.Processes {
82 if process.Role != "daemon" && process.Role != "client" {
83 t.Fatalf("run %d unexpected role %q", run, process.Role)
84 }
85 if filepath.Dir(process.TracePath) != runDir {
86 t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
87 }
88 trace, err := os.ReadFile(process.TracePath)
89 if err != nil {
90 t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
91 }
92 if len(bytes.TrimSpace(trace)) == 0 {
93 t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
94 }
95 for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
96 var record traceRecord
97 if err := json.Unmarshal(line, &record); err != nil {
98 t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
99 }
100 if record.ProcessID != process.ProcessID || record.Scenario != local.ID || record.Run != uint64(run) {
101 t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
102 }
103 }
104 }
105 if _, err := os.Stat(filepath.Join(runDir, "runtime", "vev", "daemon.sock")); !errors.Is(err, os.ErrNotExist) {
106 t.Fatalf("run %d left daemon socket for the next repetition: %v", run, err)
107 }
108 }
109}
110
111func TestHarnessResolvesCanonicalRelativePathsBeforeRoleCWDChanges(t *testing.T) {
112 if testing.Short() {
113 t.Skip("public CLI relative-path lifecycle")
114 }
115 root, err := filepath.Abs(filepath.Join("..", ".."))
116 if err != nil {
117 t.Fatal(err)
118 }
119 // Keep the real Unix-socket path below its platform limit while testing the
120 // exact documented relative output hierarchy.
121 workspace, err := os.MkdirTemp("", "vev-rel-")
122 if err != nil {
123 t.Fatal(err)
124 }
125 removeTestTree(t, workspace)
126 bin := filepath.Join(workspace, "vev")
127 build := exec.Command("go", "build", "-o", bin, "./")
128 build.Dir = root
129 if output, err := build.CombinedOutput(); err != nil {
130 t.Fatalf("build public CLI: %v\n%s", err, output)
131 }
132 manifestPath := filepath.Join(workspace, "testdata", "perf", "manifest.json")
133 if err := safedir.EnsurePrivate(filepath.Dir(manifestPath)); err != nil {
134 t.Fatal(err)
135 }
136 contents, err := os.ReadFile(filepath.Join(root, "testdata", "perf", "manifest.json"))
137 if err != nil {
138 t.Fatal(err)
139 }
140 if err := os.WriteFile(manifestPath, contents, 0o600); err != nil {
141 t.Fatal(err)
142 }
143 originalWD, err := os.Getwd()
144 if err != nil {
145 t.Fatal(err)
146 }
147 if err := os.Chdir(workspace); err != nil {
148 t.Fatal(err)
149 }
150 t.Cleanup(func() {
151 if err := os.Chdir(originalWD); err != nil {
152 t.Errorf("restore cwd: %v", err)
153 }
154 })
155
156 // These are the documented relative locations. cliLauncher changes every
157 // role to its run directory, so a failure to resolve them before launch
158 // would make VEV_PERF_TRACE point at a nonexistent nested directory.
159 h := defaultHarness()
160 h.clock = &fakeClock{}
161 if err := run([]string{
162 "--vev-bin", "./vev",
163 "--manifest", "testdata/perf/manifest.json",
164 "--out", "testdata/perf/results",
165 "--scenario", "1x4-idle-local",
166 "--warmup", "0s",
167 "--duration", "30s",
168 "--repetitions", "10",
169 }, h); err != nil {
170 t.Fatal(err)
171 }
172
173 out := filepath.Join(workspace, "testdata", "perf", "results")
174 for run := 1; run <= 10; run++ {
175 runDir := filepath.Join(out, fmt.Sprintf("1x4-idle-local-run-%03d", run))
176 manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
177 if err != nil {
178 t.Fatalf("run %d manifest: %v", run, err)
179 }
180 var persisted runManifest
181 if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
182 t.Fatalf("run %d manifest JSON: %v", run, err)
183 }
184 if persisted.Run != run || len(persisted.Processes) != 2 {
185 t.Fatalf("run %d manifest=%+v", run, persisted)
186 }
187 for _, process := range persisted.Processes {
188 if !filepath.IsAbs(process.TracePath) || filepath.Dir(process.TracePath) != runDir {
189 t.Fatalf("run %d %s trace path=%q, want absolute path in %q", run, process.Role, process.TracePath, runDir)
190 }
191 trace, err := os.ReadFile(process.TracePath)
192 if err != nil || len(bytes.TrimSpace(trace)) == 0 {
193 t.Fatalf("run %d %s trace=%q err=%v", run, process.Role, process.TracePath, err)
194 }
195 }
196 }
197}
198
199func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
200 out := t.TempDir()
201 raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
202 if err != nil {
203 t.Fatal(err)
204 }
205 closeTestFile(t, raw)
206 s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
207 stale := filepath.Join(out, "isolated-run-001")
208 if err := os.Mkdir(stale, 0o700); err != nil {
209 t.Fatal(err)
210 }
211 if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
212 t.Fatal(err)
213 }
214 launcher := &fakeLauncher{}
215 h := defaultHarness()
216 h.clock, h.launcher = &fakeClock{}, launcher
217 _, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
218 if err == nil || !strings.Contains(err.Error(), "create isolated run directory") {
219 t.Fatalf("existing run directory error = %v", err)
220 }
221 if len(launcher.mappings) != 0 {
222 t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
223 }
224 if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil || string(contents) != "stale" {
225 t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
226 }
227}
228
229func TestHarnessDoesNotInheritNestedSession(t *testing.T) {
230 env := withoutEnv([]string{"VEV=session=old", "PATH=/bin", "VEV_PERF_TRACE=old"}, "VEV")
231 if !equalStrings(env, []string{"PATH=/bin", "VEV_PERF_TRACE=old"}) {
232 t.Fatalf("environment=%q", env)
233 }
234}
235
236func TestHarnessRoutesEveryRemoteFixtureThroughItsDeclaredPeer(t *testing.T) {
237 cases := []struct {
238 name string
239 transport transport
240 peer string
241 wantRTT int
242 wantLoss int
243 }{
244 {"ssh", transport{ID: "ssh_stdio", Kind: "ssh_stdio"}, "ssh_stdio_peer", 0, 0},
245 {"udp baseline", transport{ID: "udp_baseline", Kind: "udp"}, "udp_peer", 0, 0},
246 {"udp 25ms", transport{ID: "udp_25ms", Kind: "udp", RTTMS: 25}, "udp_peer", 25, 0},
247 {"udp 100ms", transport{ID: "udp_100ms", Kind: "udp", RTTMS: 100}, "udp_peer", 100, 0},
248 {"udp loss zero", transport{ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0}, "udp_peer", 0, 0},
249 {"udp loss", transport{ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1}, "udp_peer", 0, 1},
250 }
251 for _, tc := range cases {
252 t.Run(tc.name, func(t *testing.T) {
253 s := scenario{ID: "route", Transport: tc.transport.ID}
254 client := routeRoleArgs(s, processMapping{Role: "client", Run: 2}, tc.transport)
255 peer := routeRoleArgs(s, processMapping{Role: tc.peer, Run: 2}, tc.transport)
256 if !equalStrings(client.Args, []string{"attach", "harness@127.0.0.1:perf-route-002"}) {
257 t.Fatalf("client did not use public remote attach: %q", client.Args)
258 }
259 if tc.peer == "ssh_stdio_peer" && !equalStrings(peer.Args, []string{"_stdio", "perf-route-002"}) {
260 t.Fatalf("ssh peer command=%q", peer.Args)
261 }
262 if tc.peer == "udp_peer" && !equalStrings(peer.Args, []string{"_udp-proxy", "perf-route-002"}) {
263 t.Fatalf("udp peer command=%q", peer.Args)
264 }
265 if peer.Transport.RTTMS != tc.wantRTT || peer.Transport.LossPercent != tc.wantLoss {
266 t.Fatalf("peer lost manifest network settings: %+v", peer.Transport)
267 }
268 })
269 }
270}
271
272func TestHarnessFakeRunnerRoutesClientToPeerAndCleansEveryRole(t *testing.T) {
273 for _, tc := range []struct {
274 name, transport, peer, peerCommand string
275 }{
276 {"ssh", "ssh_stdio", "ssh_stdio_peer", "_stdio"},
277 {"udp baseline", "udp_baseline", "udp_peer", "_udp-proxy"},
278 {"udp 25ms", "udp_25ms", "udp_peer", "_udp-proxy"},
279 {"udp 100ms", "udp_100ms", "udp_peer", "_udp-proxy"},
280 {"udp loss zero", "udp_loss_0pct", "udp_peer", "_udp-proxy"},
281 {"udp loss", "udp_loss_1pct", "udp_peer", "_udp-proxy"},
282 } {
283 t.Run(tc.name, func(t *testing.T) {
284 dir := t.TempDir()
285 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
286 if err != nil {
287 t.Fatal(err)
288 }
289 closeTestFile(t, raw)
290 l := &fakeLauncher{}
291 h := defaultHarness()
292 h.clock, h.launcher = &fakeClock{}, l
293 s := scenario{ID: "routed", Transport: tc.transport, Roles: []string{"daemon", "client", tc.peer}}
294 if _, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, s, 1, raw); err != nil {
295 t.Fatal(err)
296 }
297 if len(l.mappings) != 3 || l.mappings[1].Role != tc.peer || l.mappings[2].Role != "client" {
298 t.Fatalf("dependency launch order=%+v", l.mappings)
299 }
300 if l.commands[1].Args[0] != tc.peerCommand || l.commands[2].Args[0] != "attach" {
301 t.Fatalf("commands do not connect client through declared peer: %+v", l.commands)
302 }
303 for i, p := range l.process {
304 if !p.closed {
305 t.Fatalf("role %s was not cleaned up", l.mappings[i].Role)
306 }
307 }
308 })
309 }
310}
311
312func TestHarnessAggregatesRawEventsSeparatelyFromRunDispersion(t *testing.T) {
313 first := append([]int64(nil), make([]int64, minimumInIntervalEventSamples+5)...)
314 for i := range first {
315 first[i] = 1
316 }
317 second := []int64{100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1_000, 1_000}
318 all, _, _, runP50s := aggregateEventResults([]runResult{
319 {EndToEnd: first, Event: distribution{P50: 1}},
320 {EndToEnd: second, Event: distribution{P50: 100}},
321 })
322 if got, dispersion := percentiles(all), percentiles(runP50s); got.Count != 30 || got.P99 != 1_000 || dispersion.Count != 2 || dispersion.P99 != 1 {
323 t.Fatalf("event aggregation=%+v run dispersion=%+v", got, dispersion)
324 }
325}
326
327func TestHarnessRawRecordsContainEveryMarkWhileSamplesRemainFiltered(t *testing.T) {
328 dir := t.TempDir()
329 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
330 if err != nil {
331 t.Fatal(err)
332 }
333 h := defaultHarness()
334 h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
335 result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "raw-complete", Roles: []string{"daemon", "client"}}, 1, raw)
336 if err != nil {
337 t.Fatal(err)
338 }
339 if err := raw.Close(); err != nil {
340 t.Fatal(err)
341 }
342 marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
343 if got, want := len(marks), 2*(result.Samples+1); got != want {
344 t.Fatalf("raw marks=%d, want warmup plus every measured boundary=%d", got, want)
345 }
346 for i, mark := range marks[:2] {
347 if mark.Sequence != 1 || !mark.Valid || []string{"input_injected", "terminal_flushed"}[i] != mark.Kind {
348 t.Fatalf("warmup mark %d=%+v", i, mark)
349 }
350 }
351 for sequence := uint64(2); sequence <= uint64(result.Samples+1); sequence++ {
352 if marks[2*(sequence-1)].Sequence != sequence || marks[2*(sequence-1)+1].Sequence != sequence {
353 t.Fatalf("raw marks did not retain both boundaries for sequence %d: %+v", sequence, marks)
354 }
355 }
356}
357
358func TestHarnessPersistsFailedFlushDiagnostics(t *testing.T) {
359 dir := t.TempDir()
360 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
361 if err != nil {
362 t.Fatal(err)
363 }
364 failure := errors.New("flush failed")
365 h := defaultHarness()
366 h.clock, h.launcher = &fakeClock{}, failedFlushLauncher{err: failure}
367 _, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "failed-flush", Roles: []string{"daemon", "client"}}, 1, raw)
368 if !errors.Is(err, failure) {
369 t.Fatalf("run error=%v, want failed flush", err)
370 }
371 if err := raw.Close(); err != nil {
372 t.Fatal(err)
373 }
374 marks := readHarnessMarks(t, filepath.Join(dir, "raw.jsonl"))
375 if got, want := len(marks), 4; got != want {
376 t.Fatalf("raw marks=%d, want warmup and failed measured pair=%d: %+v", got, want, marks)
377 }
378 failed := marks[len(marks)-1]
379 if failed.Sequence != 2 || failed.Kind != "terminal_flushed" || failed.Valid {
380 t.Fatalf("failed flush diagnostic=%+v", failed)
381 }
382}
383
384func TestHarnessSchedulesMeasuredEventsAcrossFullInterval(t *testing.T) {
385 dir := t.TempDir()
386 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
387 if err != nil {
388 t.Fatal(err)
389 }
390 closeTestFile(t, raw)
391 h := defaultHarness()
392 h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
393 result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "schedule", Roles: []string{"daemon", "client"}}, 1, raw)
394 if err != nil {
395 t.Fatal(err)
396 }
397 if result.Samples < minimumInIntervalEventSamples || result.Cadence.Count != result.Samples-1 || result.MaxGap <= 0 {
398 t.Fatalf("result did not include repeated measured events: %+v", result)
399 }
400 if err := raw.Close(); err != nil {
401 t.Fatal(err)
402 }
403 records, err := readJSONL(filepath.Join(dir, "raw.jsonl"))
404 if err != nil {
405 t.Fatal(err)
406 }
407 if want := result.Samples*2 + 2; len(records) != want {
408 t.Fatalf("raw records=%d, want %d warmup and measured boundaries", len(records), want)
409 }
410}
411
412func TestHarnessPreservesWorkloadAndCleanupErrors(t *testing.T) {
413 dir := t.TempDir()
414 raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
415 if err != nil {
416 t.Fatal(err)
417 }
418 closeTestFile(t, raw)
419 primary := errors.New("primary workload failure")
420 cleanup := errors.New("daemon cleanup failure")
421 launcher := &fakeLauncher{
422 measureErr: map[string]error{"client": primary},
423 closeErr: map[string]error{"daemon": cleanup},
424 }
425 h := defaultHarness()
426 h.clock, h.launcher = &fakeClock{}, launcher
427 _, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "errors", Roles: []string{"daemon", "client"}}, 1, raw)
428 if !errors.Is(err, primary) || !errors.Is(err, cleanup) {
429 t.Fatalf("workload and cleanup errors were not both preserved: %v", err)
430 }
431 if got, want := err.Error(), primary.Error()+"\nclose daemon role: "+cleanup.Error(); got != want {
432 t.Fatalf("error ordering=%q want %q", got, want)
433 }
434 for i, p := range launcher.process {
435 if !p.closed {
436 t.Fatalf("role %s was not closed", launcher.mappings[i].Role)
437 }
438 }
439}
440
441func TestHarnessCleansRunDirectoryOnProcessFailure(t *testing.T) {
442 dir := t.TempDir()
443 raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
444 if err != nil {
445 t.Fatal(err)
446 }
447 closeTestFile(t, raw)
448 launcher := &fakeLauncher{}
449 h := defaultHarness()
450 h.clock = &fakeClock{}
451 // The client is the second role and fails after the daemon has started.
452 h.launcher = &failingLauncher{fakeLauncher: launcher}
453 o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
454 if _, err := h.runOne(o, manifest{}, scenario{ID: "cleanup", Roles: []string{"daemon", "client"}}, 1, raw); err == nil {
455 t.Fatal("failed process accepted")
456 }
457 if _, err := os.Stat(filepath.Join(dir, "cleanup-run-001")); !os.IsNotExist(err) {
458 t.Fatalf("failed run directory remains: %v", err)
459 }
460}