1// SPDX-License-Identifier: MIT
2
3package expr
4
5import (
6 "encoding/json"
7 "sort"
8 "testing"
9)
10
11// knownRunnerDifference classifies the answers of the corpus on which this
12// package knowingly differs from the runner. It returns the reason, or "".
13// There is none left: negative zero is written "-0" like the legacy engine.
14func knownRunnerDifference(g, r runnerCase) string { return "" }
15
16// TestRunnerOracle holds the package to the answers of the runner's own code on
17// the 34 765 expressions of the oracle corpus: same result kind, string form
18// and toJSON text, same error phase and kind (the error texts of the runner's
19// Json.NET are not reproduced).
20func TestRunnerOracle(t *testing.T) {
21 of, ctxValues, ctxNames := loadOracle(t)
22 rf := loadRunner(t)
23 if len(rf.Cases) != len(of.Cases) {
24 t.Fatalf("runner answers %d cases, oracle has %d", len(rf.Cases), len(of.Cases))
25 }
26 t.Logf("runner: %s @ %s, %s", rf.Reference.Source, rf.Reference.Commit, rf.Reference.Runtime)
27 ec := EvalContext{Contexts: ctxValues, Funcs: oracleFuncs, MaxResultBytes: -1}
28 known := map[string]int{}
29 failures := 0
30 for i, rc := range rf.Cases {
31 if rc.E != of.Cases[i].E {
32 t.Fatalf("case %d: runner expression %q, oracle %q", i, rc.E, of.Cases[i].E)
33 }
34 g := goAnswer(rc.E, ctxNames, ec)
35 if sameAnswer(g, rc) {
36 continue
37 }
38 if reason := knownRunnerDifference(g, rc); reason != "" {
39 known[reason]++
40 continue
41 }
42 failures++
43 if failures <= 30 {
44 gj, _ := json.Marshal(g)
45 rj, _ := json.Marshal(rc)
46 t.Errorf("%q:\n go %s\n runner %s", clip(rc.E), gj, rj)
47 }
48 }
49 reasons := make([]string, 0, len(known))
50 for k := range known {
51 reasons = append(reasons, k)
52 }
53 sort.Strings(reasons)
54 for _, k := range reasons {
55 t.Logf("known difference from the runner (%d cases): %s", known[k], k)
56 }
57 t.Logf("%d cases, %d failures", len(rf.Cases), failures)
58 if failures > 0 {
59 t.Fatalf("%d mismatches with the runner", failures)
60 }
61}