1//go:build linux
2
3package main
4
5import (
6 "encoding/json"
7 "errors"
8 "os"
9 "path/filepath"
10 "strings"
11 "testing"
12 "time"
13)
14
15func TestHarnessScenarioSelectionKeepsCanonicalManifestValidation(t *testing.T) {
16 root, err := filepath.Abs(filepath.Join("..", ".."))
17 if err != nil {
18 t.Fatal(err)
19 }
20 manifestPath := filepath.Join(root, "testdata", "perf", "manifest.json")
21 m, err := readManifest(manifestPath)
22 if err != nil {
23 t.Fatal(err)
24 }
25 selected := m.Scenarios[0]
26 h := defaultHarness()
27 h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
28 out := filepath.Join(t.TempDir(), "out")
29 if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
30 t.Fatal(err)
31 }
32 var runs []runResult
33 b, err := os.ReadFile(filepath.Join(out, "runs.json"))
34 if err != nil {
35 t.Fatal(err)
36 }
37 if err := json.Unmarshal(b, &runs); err != nil {
38 t.Fatal(err)
39 }
40 if len(runs) != 10 {
41 t.Fatalf("selected canonical runs=%d, want 10", len(runs))
42 }
43 for _, result := range runs {
44 if result.Scenario != selected.ID {
45 t.Fatalf("selected scenario = %q, want %q", result.Scenario, selected.ID)
46 }
47 }
48}
49
50func TestHarnessScenarioSelectionRejectsIncompleteCanonicalManifest(t *testing.T) {
51 m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
52 if err != nil {
53 t.Fatal(err)
54 }
55 selected := m.Scenarios[0]
56 m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
57 dir := t.TempDir()
58 manifestPath := filepath.Join(dir, "manifest.json")
59 if err := os.WriteFile(manifestPath, []byte(mustJSON(m)), 0o600); err != nil {
60 t.Fatal(err)
61 }
62 launcher := &fakeLauncher{}
63 h := defaultHarness()
64 h.clock, h.launcher = &fakeClock{}, launcher
65 out := filepath.Join(dir, "out")
66 err = run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h)
67 if err == nil || !strings.Contains(err.Error(), "missing scenario or inapplicable reason") {
68 t.Fatalf("incomplete manifest selection error = %v", err)
69 }
70 if len(launcher.mappings) != 0 {
71 t.Fatalf("launched with incomplete canonical manifest: %+v", launcher.mappings)
72 }
73 if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
74 t.Fatalf("output created before manifest validation: %v", statErr)
75 }
76}
77
78func TestResolvePathOptionsMakesRelativeAndAbsoluteInputsIdentical(t *testing.T) {
79 cwd, err := os.Getwd()
80 if err != nil {
81 t.Fatal(err)
82 }
83 absolute := options{
84 vevBin: filepath.Join(cwd, "vev"),
85 manifest: filepath.Join(cwd, "testdata", "perf", "manifest.json"),
86 out: filepath.Join(cwd, "testdata", "perf", "results"),
87 }
88 relative := options{
89 vevBin: "vev",
90 manifest: "testdata/perf/manifest.json",
91 out: "testdata/perf/results",
92 }
93 gotRelative, err := resolvePathOptions(relative)
94 if err != nil {
95 t.Fatal(err)
96 }
97 gotAbsolute, err := resolvePathOptions(absolute)
98 if err != nil {
99 t.Fatal(err)
100 }
101 if gotRelative != gotAbsolute {
102 t.Fatalf("relative paths resolved to %+v, absolute paths to %+v", gotRelative, gotAbsolute)
103 }
104}
105
106func TestHarnessFlagsRejectShortMeasurements(t *testing.T) {
107 for _, args := range [][]string{
108 {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "29s", "--repetitions", "10"},
109 {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "9"},
110 {"--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"},
111 } {
112 if _, err := parseOptions(args); err == nil {
113 t.Fatalf("parseOptions(%v) succeeded", args)
114 }
115 }
116 if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"}); err == nil {
117 t.Fatal("missing --warmup accepted")
118 }
119 if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}); err != nil {
120 t.Fatal(err)
121 }
122}
123
124func TestValidateManifestRejectsEmptyAndDuplicateMatrixIDs(t *testing.T) {
125 canonical, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
126 if err != nil {
127 t.Fatal(err)
128 }
129 for _, tc := range []struct {
130 name string
131 mutate func(*manifest)
132 want string
133 }{
134 {"empty topology", func(m *manifest) { m.Topologies[0].ID = "" }, "empty topology id"},
135 {"duplicate topology", func(m *manifest) { m.Topologies[1].ID = m.Topologies[0].ID }, "duplicate topology id"},
136 {"empty workload", func(m *manifest) { m.Workloads[0] = "" }, "empty workload id"},
137 {"duplicate workload", func(m *manifest) { m.Workloads[1] = m.Workloads[0] }, "duplicate workload id"},
138 {"empty transport", func(m *manifest) { m.Transports[0].ID = "" }, "empty transport id"},
139 {"duplicate transport", func(m *manifest) { m.Transports[1].ID = m.Transports[0].ID }, "duplicate transport id"},
140 } {
141 t.Run(tc.name, func(t *testing.T) {
142 m := canonical
143 m.Topologies = append([]topology(nil), canonical.Topologies...)
144 m.Workloads = append([]string(nil), canonical.Workloads...)
145 m.Transports = append([]transport(nil), canonical.Transports...)
146 tc.mutate(&m)
147 if err := validateManifest(m); err == nil || !strings.Contains(err.Error(), tc.want) {
148 t.Fatalf("validateManifest() error = %v, want %q", err, tc.want)
149 }
150 })
151 }
152}
153
154func TestValidateManifestRejectsFullyCoveredUnknownMatrixIDs(t *testing.T) {
155 canonical, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
156 if err != nil {
157 t.Fatal(err)
158 }
159 for _, tc := range []struct {
160 name string
161 mutate func(*manifest)
162 want string
163 }{
164 {
165 name: "topology",
166 mutate: func(m *manifest) {
167 m.Topologies = append(m.Topologies, topology{ID: "9x9", Geometry: "120x40", RowsPerPane: 10000})
168 for _, s := range canonical.Scenarios {
169 s.ID = "9x9-" + s.ID
170 s.Topology = "9x9"
171 m.Scenarios = append(m.Scenarios, s)
172 }
173 },
174 want: "unknown topology id \"9x9\"",
175 },
176 {
177 name: "workload",
178 mutate: func(m *manifest) {
179 m.Workloads = append(m.Workloads, "unknown_workload")
180 for _, s := range canonical.Scenarios {
181 s.ID = "unknown-workload-" + s.ID
182 s.Workload = "unknown_workload"
183 m.Scenarios = append(m.Scenarios, s)
184 }
185 },
186 want: "unknown workload id \"unknown_workload\"",
187 },
188 } {
189 t.Run(tc.name, func(t *testing.T) {
190 m := canonical
191 m.Topologies = append([]topology(nil), canonical.Topologies...)
192 m.Workloads = append([]string(nil), canonical.Workloads...)
193 m.Scenarios = append([]scenario(nil), canonical.Scenarios...)
194 tc.mutate(&m)
195 if err := validateManifest(m); err == nil || !strings.Contains(err.Error(), tc.want) {
196 t.Fatalf("validateManifest() error = %v, want %q", err, tc.want)
197 }
198 })
199 }
200}
201
202func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
203 m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
204 if err != nil {
205 t.Fatal(err)
206 }
207 if err := validateManifest(m); err != nil {
208 t.Fatal(err)
209 }
210 if len(m.Scenarios) != 4*9*7 {
211 t.Fatalf("scenarios=%d", len(m.Scenarios))
212 }
213 m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
214 if err := validateManifest(m); err == nil {
215 t.Fatal("missing combination accepted")
216 }
217}
218
219func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
220 dir := t.TempDir()
221 raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
222 if err != nil {
223 t.Fatal(err)
224 }
225 closeTestFile(t, raw)
226 launcher := &fakeLauncher{}
227 h := defaultHarness()
228 h.clock = &fakeClock{}
229 h.launcher = launcher
230 o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
231 r, err := h.runOne(o, manifest{}, scenario{ID: "4x4-local", Roles: []string{"daemon", "client"}}, 1, raw)
232 if err != nil {
233 t.Fatal(err)
234 }
235 if !launcher.manifestPresent {
236 t.Fatal("launched before complete run manifest")
237 }
238 if len(r.EndToEnd) < minimumInIntervalEventSamples || r.Samples < minimumInIntervalEventSamples {
239 t.Fatalf("samples=%+v", r)
240 }
241 seen := map[string]bool{}
242 for _, m := range r.Processes {
243 if m.ProcessID == "" || m.ClockDomain != m.ProcessID || m.TracePath == "" || seen[m.TracePath] {
244 t.Fatalf("bad mapping %+v", m)
245 }
246 seen[m.TracePath] = true
247 if _, err := os.Stat(m.TracePath); err != nil {
248 t.Fatal(err)
249 }
250 }
251 b, err := os.ReadFile(filepath.Join(dir, "4x4-local-run-001", "manifest.json"))
252 if err != nil {
253 t.Fatal(err)
254 }
255 var got runManifest
256 if err := json.Unmarshal(b, &got); err != nil {
257 t.Fatal(err)
258 }
259 if len(got.Processes) != 2 {
260 t.Fatalf("manifest=%+v", got)
261 }
262 if len(launcher.process) != 2 || !launcher.process[0].closed || len(launcher.process[0].warmups) != 0 || len(launcher.process[1].warmups) != 1 || len(launcher.process[1].measures) < minimumInIntervalEventSamples {
263 t.Fatalf("process lifecycle not recorded: %+v", launcher.process)
264 }
265}
266
267func TestHarnessRejectsMissingManifestRoleTrace(t *testing.T) {
268 dir := t.TempDir()
269 missing := filepath.Join(dir, "daemon.jsonl")
270 client := filepath.Join(dir, "client.jsonl")
271 if err := os.WriteFile(client, nil, 0o600); err != nil {
272 t.Fatal(err)
273 }
274 _, err := mergeProcessTraces([]processMapping{
275 {ProcessID: "daemon", ClockDomain: "daemon", TracePath: missing, Role: "daemon", Scenario: "s", Run: 1},
276 {ProcessID: "client", ClockDomain: "client", TracePath: client, Role: "client", Scenario: "s", Run: 1},
277 })
278 if !errors.Is(err, os.ErrNotExist) || !strings.Contains(err.Error(), "daemon role") {
279 t.Fatalf("missing daemon trace error = %v, want daemon role not-exist error", err)
280 }
281}