detached f90898d

test(obs): validate canonical local evidence

commit
f90898dbae13bacefb55069be755885d0bd988fc
author
brice · · 2026-07-13 01:20 UTC
parent
99293b6
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
2 files · +100 -4
Unified
UnifiedSplit
M cmd/vev-perf-harness/main.go +21 -3
@@ -821,6 +821,7 @@ type harness struct {
821821 clock clock
822822 launcher launcher
823823 mkdir func(string) error
824+ createRunDir func(string) error
824825 create func(string) (*os.File, error)
825826 removeAll func(string) error
826827 gitSHA func() string
@@ -834,7 +835,20 @@ func main() {
834835 }
835836 }
836837 func defaultHarness() *harness {
837- return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}
838+ return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, createRunDir: createExclusiveRunDir, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}
839+}
840+
841+// createExclusiveRunDir prevents stale runtime, socket, state, or trace files
842+// from an earlier invocation being mistaken for the next repetition's evidence.
843+func createExclusiveRunDir(path string) error {
844+ if err := os.Mkdir(path, 0o700); err != nil {
845+ return err
846+ }
847+ if err := safedir.EnsurePrivate(path); err != nil {
848+ _ = os.Remove(path)
849+ return err
850+ }
851+ return nil
838852 }
839853
840854 // recordedGitSHA prefers build provenance so an installed public harness does
@@ -1108,8 +1122,12 @@ func set(v []string) map[string]bool {
11081122 }
11091123 func (h *harness) runOne(o options, mat manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {
11101124 dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))
1111- if err = h.mkdir(dir); err != nil {
1112- return res, err
1125+ createRunDir := h.createRunDir
1126+ if createRunDir == nil {
1127+ createRunDir = createExclusiveRunDir
1128+ }
1129+ if err = createRunDir(dir); err != nil {
1130+ return res, fmt.Errorf("create isolated run directory: %w", err)
11131131 }
11141132 success := false
11151133 defer func() {
@@ -821,6 +821,7 @@ type harness struct {
821 clock clock821 clock clock
822 launcher launcher822 launcher launcher
823 mkdir func(string) error823 mkdir func(string) error
824+ createRunDir func(string) error
824 create func(string) (*os.File, error)825 create func(string) (*os.File, error)
825 removeAll func(string) error826 removeAll func(string) error
826 gitSHA func() string827 gitSHA func() string
@@ -834,7 +835,20 @@ func main() {
834 }835 }
835 }836 }
836 func defaultHarness() *harness {837 func defaultHarness() *harness {
837- return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}838+ return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, createRunDir: createExclusiveRunDir, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}
839+}
840+
841+// createExclusiveRunDir prevents stale runtime, socket, state, or trace files
842+// from an earlier invocation being mistaken for the next repetition's evidence.
843+func createExclusiveRunDir(path string) error {
844+ if err := os.Mkdir(path, 0o700); err != nil {
845+ return err
846+ }
847+ if err := safedir.EnsurePrivate(path); err != nil {
848+ _ = os.Remove(path)
849+ return err
850+ }
851+ return nil
838 }852 }
839 853
840 // recordedGitSHA prefers build provenance so an installed public harness does854 // recordedGitSHA prefers build provenance so an installed public harness does
@@ -1108,8 +1122,12 @@ func set(v []string) map[string]bool {
1108 }1122 }
1109 func (h *harness) runOne(o options, mat manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {1123 func (h *harness) runOne(o options, mat manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {
1110 dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))1124 dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))
1111- if err = h.mkdir(dir); err != nil {1125+ createRunDir := h.createRunDir
1112- return res, err1126+ if createRunDir == nil {
1127+ createRunDir = createExclusiveRunDir
1128+ }
1129+ if err = createRunDir(dir); err != nil {
1130+ return res, fmt.Errorf("create isolated run directory: %w", err)
1113 }1131 }
1114 success := false1132 success := false
1115 defer func() {1133 defer func() {
M cmd/vev-perf-harness/main_test.go +79 -1
@@ -4,6 +4,7 @@ import (
44 "bytes"
55 "encoding/json"
66 "errors"
7+ "fmt"
78 "net"
89 "os"
910 "os/exec"
@@ -249,13 +250,44 @@ func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
249250 if len(result.Processes) != 2 {
250251 t.Fatalf("run %d process mappings = %+v", run, result.Processes)
251252 }
253+ runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
254+ manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
255+ if err != nil {
256+ t.Fatalf("run %d manifest: %v", run, err)
257+ }
258+ var persisted runManifest
259+ if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
260+ t.Fatalf("run %d manifest JSON: %v", run, err)
261+ }
262+ if persisted.Run != run || len(persisted.Processes) != len(result.Processes) {
263+ t.Fatalf("run %d manifest = %+v", run, persisted)
264+ }
252265 for _, process := range result.Processes {
253266 if process.Role != "daemon" && process.Role != "client" {
254267 t.Fatalf("run %d unexpected role %q", run, process.Role)
255268 }
256- if _, err := os.Stat(process.TracePath); err != nil {
269+ if filepath.Dir(process.TracePath) != runDir {
270+ t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
271+ }
272+ trace, err := os.ReadFile(process.TracePath)
273+ if err != nil {
257274 t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
258275 }
276+ if len(bytes.TrimSpace(trace)) == 0 {
277+ t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
278+ }
279+ for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
280+ var record traceRecord
281+ if err := json.Unmarshal(line, &record); err != nil {
282+ t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
283+ }
284+ if record.ProcessID != process.ProcessID || record.Scenario != local.ID || record.Run != uint64(run) {
285+ t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
286+ }
287+ }
288+ }
289+ if _, err := os.Stat(filepath.Join(runDir, "runtime", "vev", "daemon.sock")); !errors.Is(err, os.ErrNotExist) {
290+ t.Fatalf("run %d left daemon socket for the next repetition: %v", run, err)
259291 }
260292 }
261293 }
@@ -368,6 +400,36 @@ func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
368400 }
369401 }
370402
403+func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
404+ out := t.TempDir()
405+ raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
406+ if err != nil {
407+ t.Fatal(err)
408+ }
409+ defer raw.Close()
410+ s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
411+ stale := filepath.Join(out, "isolated-run-001")
412+ if err := os.Mkdir(stale, 0o700); err != nil {
413+ t.Fatal(err)
414+ }
415+ if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
416+ t.Fatal(err)
417+ }
418+ launcher := &fakeLauncher{}
419+ h := defaultHarness()
420+ h.clock, h.launcher = &fakeClock{}, launcher
421+ _, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
422+ if err == nil || !strings.Contains(err.Error(), "create isolated run directory") {
423+ t.Fatalf("existing run directory error = %v", err)
424+ }
425+ if len(launcher.mappings) != 0 {
426+ t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
427+ }
428+ if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil || string(contents) != "stale" {
429+ t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
430+ }
431+}
432+
371433 func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
372434 dir := t.TempDir()
373435 raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
@@ -1198,6 +1260,22 @@ func TestHarnessAcceptsConcurrentProductionTraceInterleaving(t *testing.T) {
11981260 }
11991261 }
12001262
1263+func TestHarnessRejectsMissingManifestRoleTrace(t *testing.T) {
1264+ dir := t.TempDir()
1265+ missing := filepath.Join(dir, "daemon.jsonl")
1266+ client := filepath.Join(dir, "client.jsonl")
1267+ if err := os.WriteFile(client, nil, 0o600); err != nil {
1268+ t.Fatal(err)
1269+ }
1270+ _, err := mergeProcessTraces([]processMapping{
1271+ {ProcessID: "daemon", ClockDomain: "daemon", TracePath: missing, Role: "daemon", Scenario: "s", Run: 1},
1272+ {ProcessID: "client", ClockDomain: "client", TracePath: client, Role: "client", Scenario: "s", Run: 1},
1273+ })
1274+ if !errors.Is(err, os.ErrNotExist) {
1275+ t.Fatalf("missing daemon trace error = %v, want not exist", err)
1276+ }
1277+}
1278+
12011279 func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {
12021280 write := func(t *testing.T, records ...traceRecord) processMapping {
12031281 t.Helper()
@@ -4,6 +4,7 @@ import (
4 "bytes"4 "bytes"
5 "encoding/json"5 "encoding/json"
6 "errors"6 "errors"
7+ "fmt"
7 "net"8 "net"
8 "os"9 "os"
9 "os/exec"10 "os/exec"
@@ -249,13 +250,44 @@ func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
249 if len(result.Processes) != 2 {250 if len(result.Processes) != 2 {
250 t.Fatalf("run %d process mappings = %+v", run, result.Processes)251 t.Fatalf("run %d process mappings = %+v", run, result.Processes)
251 }252 }
253+ runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
254+ manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
255+ if err != nil {
256+ t.Fatalf("run %d manifest: %v", run, err)
257+ }
258+ var persisted runManifest
259+ if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
260+ t.Fatalf("run %d manifest JSON: %v", run, err)
261+ }
262+ if persisted.Run != run || len(persisted.Processes) != len(result.Processes) {
263+ t.Fatalf("run %d manifest = %+v", run, persisted)
264+ }
252 for _, process := range result.Processes {265 for _, process := range result.Processes {
253 if process.Role != "daemon" && process.Role != "client" {266 if process.Role != "daemon" && process.Role != "client" {
254 t.Fatalf("run %d unexpected role %q", run, process.Role)267 t.Fatalf("run %d unexpected role %q", run, process.Role)
255 }268 }
256- if _, err := os.Stat(process.TracePath); err != nil {269+ if filepath.Dir(process.TracePath) != runDir {
270+ t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
271+ }
272+ trace, err := os.ReadFile(process.TracePath)
273+ if err != nil {
257 t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)274 t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
258 }275 }
276+ if len(bytes.TrimSpace(trace)) == 0 {
277+ t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
278+ }
279+ for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
280+ var record traceRecord
281+ if err := json.Unmarshal(line, &record); err != nil {
282+ t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
283+ }
284+ if record.ProcessID != process.ProcessID || record.Scenario != local.ID || record.Run != uint64(run) {
285+ t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
286+ }
287+ }
288+ }
289+ if _, err := os.Stat(filepath.Join(runDir, "runtime", "vev", "daemon.sock")); !errors.Is(err, os.ErrNotExist) {
290+ t.Fatalf("run %d left daemon socket for the next repetition: %v", run, err)
259 }291 }
260 }292 }
261 }293 }
@@ -368,6 +400,36 @@ func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
368 }400 }
369 }401 }
370 402
403+func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
404+ out := t.TempDir()
405+ raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
406+ if err != nil {
407+ t.Fatal(err)
408+ }
409+ defer raw.Close()
410+ s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
411+ stale := filepath.Join(out, "isolated-run-001")
412+ if err := os.Mkdir(stale, 0o700); err != nil {
413+ t.Fatal(err)
414+ }
415+ if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
416+ t.Fatal(err)
417+ }
418+ launcher := &fakeLauncher{}
419+ h := defaultHarness()
420+ h.clock, h.launcher = &fakeClock{}, launcher
421+ _, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
422+ if err == nil || !strings.Contains(err.Error(), "create isolated run directory") {
423+ t.Fatalf("existing run directory error = %v", err)
424+ }
425+ if len(launcher.mappings) != 0 {
426+ t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
427+ }
428+ if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil || string(contents) != "stale" {
429+ t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
430+ }
431+}
432+
371 func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {433 func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
372 dir := t.TempDir()434 dir := t.TempDir()
373 raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)435 raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
@@ -1198,6 +1260,22 @@ func TestHarnessAcceptsConcurrentProductionTraceInterleaving(t *testing.T) {
1198 }1260 }
1199 }1261 }
1200 1262
1263+func TestHarnessRejectsMissingManifestRoleTrace(t *testing.T) {
1264+ dir := t.TempDir()
1265+ missing := filepath.Join(dir, "daemon.jsonl")
1266+ client := filepath.Join(dir, "client.jsonl")
1267+ if err := os.WriteFile(client, nil, 0o600); err != nil {
1268+ t.Fatal(err)
1269+ }
1270+ _, err := mergeProcessTraces([]processMapping{
1271+ {ProcessID: "daemon", ClockDomain: "daemon", TracePath: missing, Role: "daemon", Scenario: "s", Run: 1},
1272+ {ProcessID: "client", ClockDomain: "client", TracePath: client, Role: "client", Scenario: "s", Run: 1},
1273+ })
1274+ if !errors.Is(err, os.ErrNotExist) {
1275+ t.Fatalf("missing daemon trace error = %v, want not exist", err)
1276+ }
1277+}
1278+
1201 func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {1279 func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {
1202 write := func(t *testing.T, records ...traceRecord) processMapping {1280 write := func(t *testing.T, records ...traceRecord) processMapping {
1203 t.Helper()1281 t.Helper()

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close