test(expr): excuse an oracle mismatch only where the C# runner gives this package's answer
- commit
- f5fe2766e40075b79af7258ff7a3f729cd761747
- author
- brice · · 2026-10-12 17:28 UTC
- parent
- 3251a55
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
2 files · +160 -15
M expr/oracle_test.go +40 -15
| @@ -128,17 +128,37 @@ type oracleRun struct { | |||
| 128 | 128 | ctxNames []string | |
| 129 | 129 | failures int | |
| 130 | 130 | c oracleCase | |
| 131 | + | // pending holds the mismatches of the current case until the case is | |
| 132 | + | // settled: a mismatch with the JavaScript reference is excused when the | |
| 133 | + | // C# runner, which is what runs on github.com, gives this package's answer. | |
| 134 | + | pending []string | |
| 131 | 135 | } | |
| 132 | 136 | ||
| 133 | 137 | func (r *oracleRun) fail(format string, args ...any) { | |
| 134 | - | r.failures++ | |
| 135 | - | if r.failures <= 40 { | |
| 136 | - | e := r.c.E | |
| 137 | - | if len(e) > 200 { | |
| 138 | - | e = e[:200] + "..." | |
| 138 | + | r.pending = append(r.pending, fmt.Sprintf(format, args...)) | |
| 139 | + | } | |
| 140 | + | ||
| 141 | + | // settle reports the pending mismatches of the current case, unless the | |
| 142 | + | // runner agrees with this package. It returns true when the case was excused. | |
| 143 | + | func (r *oracleRun) settle(rc runnerCase, ctxNames []string) (excused bool) { | |
| 144 | + | if len(r.pending) == 0 { | |
| 145 | + | return false | |
| 146 | + | } | |
| 147 | + | defer func() { r.pending = r.pending[:0] }() | |
| 148 | + | if rc.E == r.c.E && sameAnswer(goAnswer(r.c.E, ctxNames, r.ec), rc) { | |
| 149 | + | return true | |
| 150 | + | } | |
| 151 | + | for _, m := range r.pending { | |
| 152 | + | r.failures++ | |
| 153 | + | if r.failures <= 40 { | |
| 154 | + | e := r.c.E | |
| 155 | + | if len(e) > 200 { | |
| 156 | + | e = e[:200] + "..." | |
| 157 | + | } | |
| 158 | + | r.t.Errorf("%q: %s", e, m) | |
| 139 | 159 | } | |
| 140 | - | r.t.Errorf("%q: %s", e, fmt.Sprintf(format, args...)) | |
| 141 | 160 | } | |
| 161 | + | return false | |
| 142 | 162 | } | |
| 143 | 163 | ||
| 144 | 164 | // checkLex compares the tokens and reports whether the case lexes. | |
| @@ -264,8 +284,13 @@ func TestOracle(t *testing.T) { | |||
| 264 | 284 | t.Logf("reference: %s, integrity %s, generated with node %s (V8 %s, Unicode %s)", ref.Package, ref.Integrity, ref.Node, ref.V8, ref.Unicode) | |
| 265 | 285 | ||
| 266 | 286 | run := &oracleRun{t: t, ctxNames: ctxNames, ec: EvalContext{Contexts: ctxValues, Funcs: oracleFuncs, MaxResultBytes: -1}} | |
| 287 | + | rf := loadRunner(t) | |
| 288 | + | if len(rf.Cases) != len(of.Cases) { | |
| 289 | + | t.Fatalf("runner answers %d cases, oracle has %d", len(rf.Cases), len(of.Cases)) | |
| 290 | + | } | |
| 267 | 291 | var counts struct{ lexErr, parseErr, evalErr, ok, empty int } | |
| 268 | - | for _, c := range of.Cases { | |
| 292 | + | excused := 0 | |
| 293 | + | for i, c := range of.Cases { | |
| 269 | 294 | run.c = c | |
| 270 | 295 | switch { | |
| 271 | 296 | case c.LexErr != nil: | |
| @@ -279,17 +304,17 @@ func TestOracle(t *testing.T) { | |||
| 279 | 304 | default: | |
| 280 | 305 | counts.ok++ | |
| 281 | 306 | } | |
| 282 | - | if !run.checkLex() { | |
| 283 | - | continue | |
| 307 | + | if run.checkLex() { | |
| 308 | + | if node := run.checkParse(); node != nil { | |
| 309 | + | run.checkEval(node) | |
| 310 | + | } | |
| 284 | 311 | } | |
| 285 | - | node := run.checkParse() | |
| 286 | - | if node == nil { | |
| 287 | - | continue | |
| 312 | + | if run.settle(rf.Cases[i], ctxNames) { | |
| 313 | + | excused++ | |
| 288 | 314 | } | |
| 289 | - | run.checkEval(node) | |
| 290 | 315 | } | |
| 291 | - | t.Logf("%d cases: %d lex errors, %d parse errors, %d empty, %d eval errors, %d values; %d failures", | |
| 292 | - | len(of.Cases), counts.lexErr, counts.parseErr, counts.empty, counts.evalErr, counts.ok, run.failures) | |
| 316 | + | t.Logf("%d cases: %d lex errors, %d parse errors, %d empty, %d eval errors, %d values; %d differ from the JavaScript where the C# runner agrees with this package; %d failures", | |
| 317 | + | len(of.Cases), counts.lexErr, counts.parseErr, counts.empty, counts.evalErr, counts.ok, excused, run.failures) | |
| 293 | 318 | if run.failures > 0 { | |
| 294 | 319 | t.Fatalf("%d oracle mismatches", run.failures) | |
| 295 | 320 | } | |
| @@ -128,17 +128,37 @@ type oracleRun struct { | |||||
| 128 | ctxNames []string | 128 | ctxNames []string | ||
| 129 | failures int | 129 | failures int | ||
| 130 | c oracleCase | 130 | c oracleCase | ||
| 131 | + | // pending holds the mismatches of the current case until the case is | |||
| 132 | + | // settled: a mismatch with the JavaScript reference is excused when the | |||
| 133 | + | // C# runner, which is what runs on github.com, gives this package's answer. | |||
| 134 | + | pending []string | |||
| 131 | } | 135 | } | ||
| 132 | 136 | ||||
| 133 | func (r *oracleRun) fail(format string, args ...any) { | 137 | func (r *oracleRun) fail(format string, args ...any) { | ||
| 134 | - | r.failures++ | 138 | + | r.pending = append(r.pending, fmt.Sprintf(format, args...)) |
| 135 | - | if r.failures <= 40 { | 139 | + | } |
| 136 | - | e := r.c.E | 140 | + | |
| 137 | - | if len(e) > 200 { | 141 | + | // settle reports the pending mismatches of the current case, unless the |
| 138 | - | e = e[:200] + "..." | 142 | + | // runner agrees with this package. It returns true when the case was excused. |
| 143 | + | func (r *oracleRun) settle(rc runnerCase, ctxNames []string) (excused bool) { | |||
| 144 | + | if len(r.pending) == 0 { | |||
| 145 | + | return false | |||
| 146 | + | } | |||
| 147 | + | defer func() { r.pending = r.pending[:0] }() | |||
| 148 | + | if rc.E == r.c.E && sameAnswer(goAnswer(r.c.E, ctxNames, r.ec), rc) { | |||
| 149 | + | return true | |||
| 150 | + | } | |||
| 151 | + | for _, m := range r.pending { | |||
| 152 | + | r.failures++ | |||
| 153 | + | if r.failures <= 40 { | |||
| 154 | + | e := r.c.E | |||
| 155 | + | if len(e) > 200 { | |||
| 156 | + | e = e[:200] + "..." | |||
| 157 | + | } | |||
| 158 | + | r.t.Errorf("%q: %s", e, m) | |||
| 139 | } | 159 | } | ||
| 140 | - | r.t.Errorf("%q: %s", e, fmt.Sprintf(format, args...)) | |||
| 141 | } | 160 | } | ||
| 161 | + | return false | |||
| 142 | } | 162 | } | ||
| 143 | 163 | ||||
| 144 | // checkLex compares the tokens and reports whether the case lexes. | 164 | // checkLex compares the tokens and reports whether the case lexes. | ||
| @@ -264,8 +284,13 @@ func TestOracle(t *testing.T) { | |||||
| 264 | t.Logf("reference: %s, integrity %s, generated with node %s (V8 %s, Unicode %s)", ref.Package, ref.Integrity, ref.Node, ref.V8, ref.Unicode) | 284 | t.Logf("reference: %s, integrity %s, generated with node %s (V8 %s, Unicode %s)", ref.Package, ref.Integrity, ref.Node, ref.V8, ref.Unicode) | ||
| 265 | 285 | ||||
| 266 | run := &oracleRun{t: t, ctxNames: ctxNames, ec: EvalContext{Contexts: ctxValues, Funcs: oracleFuncs, MaxResultBytes: -1}} | 286 | run := &oracleRun{t: t, ctxNames: ctxNames, ec: EvalContext{Contexts: ctxValues, Funcs: oracleFuncs, MaxResultBytes: -1}} | ||
| 287 | + | rf := loadRunner(t) | |||
| 288 | + | if len(rf.Cases) != len(of.Cases) { | |||
| 289 | + | t.Fatalf("runner answers %d cases, oracle has %d", len(rf.Cases), len(of.Cases)) | |||
| 290 | + | } | |||
| 267 | var counts struct{ lexErr, parseErr, evalErr, ok, empty int } | 291 | var counts struct{ lexErr, parseErr, evalErr, ok, empty int } | ||
| 268 | - | for _, c := range of.Cases { | 292 | + | excused := 0 |
| 293 | + | for i, c := range of.Cases { | |||
| 269 | run.c = c | 294 | run.c = c | ||
| 270 | switch { | 295 | switch { | ||
| 271 | case c.LexErr != nil: | 296 | case c.LexErr != nil: | ||
| @@ -279,17 +304,17 @@ func TestOracle(t *testing.T) { | |||||
| 279 | default: | 304 | default: | ||
| 280 | counts.ok++ | 305 | counts.ok++ | ||
| 281 | } | 306 | } | ||
| 282 | - | if !run.checkLex() { | 307 | + | if run.checkLex() { |
| 283 | - | continue | 308 | + | if node := run.checkParse(); node != nil { |
| 309 | + | run.checkEval(node) | |||
| 310 | + | } | |||
| 284 | } | 311 | } | ||
| 285 | - | node := run.checkParse() | 312 | + | if run.settle(rf.Cases[i], ctxNames) { |
| 286 | - | if node == nil { | 313 | + | excused++ |
| 287 | - | continue | |||
| 288 | } | 314 | } | ||
| 289 | - | run.checkEval(node) | |||
| 290 | } | 315 | } | ||
| 291 | - | t.Logf("%d cases: %d lex errors, %d parse errors, %d empty, %d eval errors, %d values; %d failures", | 316 | + | t.Logf("%d cases: %d lex errors, %d parse errors, %d empty, %d eval errors, %d values; %d differ from the JavaScript where the C# runner agrees with this package; %d failures", |
| 292 | - | len(of.Cases), counts.lexErr, counts.parseErr, counts.empty, counts.evalErr, counts.ok, run.failures) | 317 | + | len(of.Cases), counts.lexErr, counts.parseErr, counts.empty, counts.evalErr, counts.ok, excused, run.failures) |
| 293 | if run.failures > 0 { | 318 | if run.failures > 0 { | ||
| 294 | t.Fatalf("%d oracle mismatches", run.failures) | 319 | t.Fatalf("%d oracle mismatches", run.failures) | ||
| 295 | } | 320 | } | ||
A expr/runner_oracle_test.go +120 -0
| @@ -0,0 +1,120 @@ | |||
| 1 | + | // SPDX-License-Identifier: MIT | |
| 2 | + | ||
| 3 | + | package expr | |
| 4 | + | ||
| 5 | + | import ( | |
| 6 | + | "compress/gzip" | |
| 7 | + | "context" | |
| 8 | + | "encoding/json" | |
| 9 | + | "os" | |
| 10 | + | "strings" | |
| 11 | + | "testing" | |
| 12 | + | ) | |
| 13 | + | ||
| 14 | + | // testdata/runner.json.gz holds what the C# runner's expression engine | |
| 15 | + | // (actions/runner, src/Sdk/DTExpressions2) answers on the corpus of | |
| 16 | + | // testdata/oracle.json.gz. It is produced by gen/runner. Where the runner and | |
| 17 | + | // the JavaScript reference disagree, the runner is what users see at run time | |
| 18 | + | // and this package follows it (see PORTING.md). TestOracle excuses a mismatch | |
| 19 | + | // with the JavaScript only if this package gives the runner's answer there. | |
| 20 | + | ||
| 21 | + | type runnerFile struct { | |
| 22 | + | Reference struct { | |
| 23 | + | Source string `json:"source"` | |
| 24 | + | Commit string `json:"commit"` | |
| 25 | + | Runtime string `json:"runtime"` | |
| 26 | + | } `json:"reference"` | |
| 27 | + | Cases []runnerCase `json:"cases"` | |
| 28 | + | } | |
| 29 | + | ||
| 30 | + | type runnerCase struct { | |
| 31 | + | E string `json:"e"` | |
| 32 | + | Empty bool `json:"empty"` | |
| 33 | + | K string `json:"k"` | |
| 34 | + | S string `json:"s"` | |
| 35 | + | J string `json:"j"` | |
| 36 | + | Err *struct { | |
| 37 | + | T string `json:"t"` | |
| 38 | + | M string `json:"m"` | |
| 39 | + | } `json:"err"` | |
| 40 | + | } | |
| 41 | + | ||
| 42 | + | func loadRunner(tb testing.TB) *runnerFile { | |
| 43 | + | tb.Helper() | |
| 44 | + | f, err := os.Open("testdata/runner.json.gz") | |
| 45 | + | if err != nil { | |
| 46 | + | tb.Fatal(err) | |
| 47 | + | } | |
| 48 | + | defer func() { _ = f.Close() }() | |
| 49 | + | zr, err := gzip.NewReader(f) | |
| 50 | + | if err != nil { | |
| 51 | + | tb.Fatal(err) | |
| 52 | + | } | |
| 53 | + | var rf runnerFile | |
| 54 | + | if err := json.NewDecoder(zr).Decode(&rf); err != nil { | |
| 55 | + | tb.Fatal(err) | |
| 56 | + | } | |
| 57 | + | return &rf | |
| 58 | + | } | |
| 59 | + | ||
| 60 | + | // goAnswer evaluates e like the runner harness does and returns its answer in | |
| 61 | + | // the runner's format. | |
| 62 | + | func goAnswer(e string, ctxNames []string, ec EvalContext) runnerCase { | |
| 63 | + | rc := runnerCase{E: e} | |
| 64 | + | node, err := Parse(e, ctxNames, oracleFuncInfos) | |
| 65 | + | if err != nil { | |
| 66 | + | rc.Err = &struct { | |
| 67 | + | T string `json:"t"` | |
| 68 | + | M string `json:"m"` | |
| 69 | + | }{"parse", err.Error()} | |
| 70 | + | return rc | |
| 71 | + | } | |
| 72 | + | if node == nil { | |
| 73 | + | rc.Empty = true | |
| 74 | + | return rc | |
| 75 | + | } | |
| 76 | + | v, err := Evaluate(context.Background(), node, ec) | |
| 77 | + | if err != nil { | |
| 78 | + | rc.Err = &struct { | |
| 79 | + | T string `json:"t"` | |
| 80 | + | M string `json:"m"` | |
| 81 | + | }{"eval", err.Error()} | |
| 82 | + | return rc | |
| 83 | + | } | |
| 84 | + | rc.K = v.Kind().String() | |
| 85 | + | rc.S = CoerceString(v) | |
| 86 | + | rc.J = ToJSON(v) | |
| 87 | + | return rc | |
| 88 | + | } | |
| 89 | + | ||
| 90 | + | // sameAnswer reports whether a Go answer matches the runner's. Errors match | |
| 91 | + | // when they are of the same phase and, for parse errors, name the same kind of | |
| 92 | + | // problem (the text before the colon); the runner appends the position and | |
| 93 | + | // names the parenthesis rather than the function in arity errors, and its | |
| 94 | + | // fromJSON errors are Json.NET messages. Other evaluation errors must be equal. | |
| 95 | + | func sameAnswer(g, r runnerCase) bool { | |
| 96 | + | if (g.Err == nil) != (r.Err == nil) { | |
| 97 | + | return false | |
| 98 | + | } | |
| 99 | + | if g.Err == nil { | |
| 100 | + | return g.Empty == r.Empty && g.K == r.K && g.S == r.S && g.J == r.J | |
| 101 | + | } | |
| 102 | + | if g.Err.T != r.Err.T { | |
| 103 | + | return false | |
| 104 | + | } | |
| 105 | + | if g.Err.T == "parse" { | |
| 106 | + | desc := func(m string) string { d, _, _ := strings.Cut(m, ":"); return d } | |
| 107 | + | return desc(g.Err.M) == desc(r.Err.M) | |
| 108 | + | } | |
| 109 | + | if strings.Contains(strings.ToLower(r.E), "fromjson") { | |
| 110 | + | return true | |
| 111 | + | } | |
| 112 | + | return g.Err.M == r.Err.M | |
| 113 | + | } | |
| 114 | + | ||
| 115 | + | func clip(s string) string { | |
| 116 | + | if len(s) > 160 { | |
| 117 | + | return s[:160] + "..." | |
| 118 | + | } | |
| 119 | + | return s | |
| 120 | + | } | |
| @@ -0,0 +1,120 @@ | |||||
| 1 | + | // SPDX-License-Identifier: MIT | |||
| 2 | + | ||||
| 3 | + | package expr | |||
| 4 | + | ||||
| 5 | + | import ( | |||
| 6 | + | "compress/gzip" | |||
| 7 | + | "context" | |||
| 8 | + | "encoding/json" | |||
| 9 | + | "os" | |||
| 10 | + | "strings" | |||
| 11 | + | "testing" | |||
| 12 | + | ) | |||
| 13 | + | ||||
| 14 | + | // testdata/runner.json.gz holds what the C# runner's expression engine | |||
| 15 | + | // (actions/runner, src/Sdk/DTExpressions2) answers on the corpus of | |||
| 16 | + | // testdata/oracle.json.gz. It is produced by gen/runner. Where the runner and | |||
| 17 | + | // the JavaScript reference disagree, the runner is what users see at run time | |||
| 18 | + | // and this package follows it (see PORTING.md). TestOracle excuses a mismatch | |||
| 19 | + | // with the JavaScript only if this package gives the runner's answer there. | |||
| 20 | + | ||||
| 21 | + | type runnerFile struct { | |||
| 22 | + | Reference struct { | |||
| 23 | + | Source string `json:"source"` | |||
| 24 | + | Commit string `json:"commit"` | |||
| 25 | + | Runtime string `json:"runtime"` | |||
| 26 | + | } `json:"reference"` | |||
| 27 | + | Cases []runnerCase `json:"cases"` | |||
| 28 | + | } | |||
| 29 | + | ||||
| 30 | + | type runnerCase struct { | |||
| 31 | + | E string `json:"e"` | |||
| 32 | + | Empty bool `json:"empty"` | |||
| 33 | + | K string `json:"k"` | |||
| 34 | + | S string `json:"s"` | |||
| 35 | + | J string `json:"j"` | |||
| 36 | + | Err *struct { | |||
| 37 | + | T string `json:"t"` | |||
| 38 | + | M string `json:"m"` | |||
| 39 | + | } `json:"err"` | |||
| 40 | + | } | |||
| 41 | + | ||||
| 42 | + | func loadRunner(tb testing.TB) *runnerFile { | |||
| 43 | + | tb.Helper() | |||
| 44 | + | f, err := os.Open("testdata/runner.json.gz") | |||
| 45 | + | if err != nil { | |||
| 46 | + | tb.Fatal(err) | |||
| 47 | + | } | |||
| 48 | + | defer func() { _ = f.Close() }() | |||
| 49 | + | zr, err := gzip.NewReader(f) | |||
| 50 | + | if err != nil { | |||
| 51 | + | tb.Fatal(err) | |||
| 52 | + | } | |||
| 53 | + | var rf runnerFile | |||
| 54 | + | if err := json.NewDecoder(zr).Decode(&rf); err != nil { | |||
| 55 | + | tb.Fatal(err) | |||
| 56 | + | } | |||
| 57 | + | return &rf | |||
| 58 | + | } | |||
| 59 | + | ||||
| 60 | + | // goAnswer evaluates e like the runner harness does and returns its answer in | |||
| 61 | + | // the runner's format. | |||
| 62 | + | func goAnswer(e string, ctxNames []string, ec EvalContext) runnerCase { | |||
| 63 | + | rc := runnerCase{E: e} | |||
| 64 | + | node, err := Parse(e, ctxNames, oracleFuncInfos) | |||
| 65 | + | if err != nil { | |||
| 66 | + | rc.Err = &struct { | |||
| 67 | + | T string `json:"t"` | |||
| 68 | + | M string `json:"m"` | |||
| 69 | + | }{"parse", err.Error()} | |||
| 70 | + | return rc | |||
| 71 | + | } | |||
| 72 | + | if node == nil { | |||
| 73 | + | rc.Empty = true | |||
| 74 | + | return rc | |||
| 75 | + | } | |||
| 76 | + | v, err := Evaluate(context.Background(), node, ec) | |||
| 77 | + | if err != nil { | |||
| 78 | + | rc.Err = &struct { | |||
| 79 | + | T string `json:"t"` | |||
| 80 | + | M string `json:"m"` | |||
| 81 | + | }{"eval", err.Error()} | |||
| 82 | + | return rc | |||
| 83 | + | } | |||
| 84 | + | rc.K = v.Kind().String() | |||
| 85 | + | rc.S = CoerceString(v) | |||
| 86 | + | rc.J = ToJSON(v) | |||
| 87 | + | return rc | |||
| 88 | + | } | |||
| 89 | + | ||||
| 90 | + | // sameAnswer reports whether a Go answer matches the runner's. Errors match | |||
| 91 | + | // when they are of the same phase and, for parse errors, name the same kind of | |||
| 92 | + | // problem (the text before the colon); the runner appends the position and | |||
| 93 | + | // names the parenthesis rather than the function in arity errors, and its | |||
| 94 | + | // fromJSON errors are Json.NET messages. Other evaluation errors must be equal. | |||
| 95 | + | func sameAnswer(g, r runnerCase) bool { | |||
| 96 | + | if (g.Err == nil) != (r.Err == nil) { | |||
| 97 | + | return false | |||
| 98 | + | } | |||
| 99 | + | if g.Err == nil { | |||
| 100 | + | return g.Empty == r.Empty && g.K == r.K && g.S == r.S && g.J == r.J | |||
| 101 | + | } | |||
| 102 | + | if g.Err.T != r.Err.T { | |||
| 103 | + | return false | |||
| 104 | + | } | |||
| 105 | + | if g.Err.T == "parse" { | |||
| 106 | + | desc := func(m string) string { d, _, _ := strings.Cut(m, ":"); return d } | |||
| 107 | + | return desc(g.Err.M) == desc(r.Err.M) | |||
| 108 | + | } | |||
| 109 | + | if strings.Contains(strings.ToLower(r.E), "fromjson") { | |||
| 110 | + | return true | |||
| 111 | + | } | |||
| 112 | + | return g.Err.M == r.Err.M | |||
| 113 | + | } | |||
| 114 | + | ||||
| 115 | + | func clip(s string) string { | |||
| 116 | + | if len(s) > 160 { | |||
| 117 | + | return s[:160] + "..." | |||
| 118 | + | } | |||
| 119 | + | return s | |||
| 120 | + | } | |||