1// SPDX-License-Identifier: MIT
2
3package expr
4
5import (
6 "compress/gzip"
7 "context"
8 "encoding/json"
9 "errors"
10 "fmt"
11 "math"
12 "os"
13 "strings"
14 "testing"
15)
16
17// The oracle file holds what the reference implementation (GitHub's
18// @actions/expressions 0.3.61, run under Node) does on about 33 000
19// expressions: hand-picked edge cases and a seeded random walk over the
20// grammar. It is generated by gen/oracle.mjs. TestOracle replays it and
21// demands identical tokens, error messages and positions, result kinds,
22// string coercions and toJSON output.
23
24type oracleFile struct {
25 Reference oracleReference `json:"reference"`
26 Contexts map[string]json.RawMessage `json:"contexts"`
27 Cases []oracleCase `json:"cases"`
28}
29
30// oracleReference records where the oracle comes from.
31type oracleReference struct {
32 Package string `json:"package"`
33 Integrity string `json:"integrity"`
34 Node string `json:"node"`
35 V8 string `json:"v8"`
36 Unicode string `json:"unicode"`
37}
38
39type oracleCase struct {
40 E string `json:"e"`
41 Tokens [][]any `json:"tokens"`
42 LexErr *string `json:"lexErr"`
43 ParseErr *string `json:"parseErr"`
44 Pos []int `json:"pos"`
45 Empty bool `json:"empty"`
46 EvalErr *string `json:"evalErr"`
47 R *oracleResult `json:"r"`
48}
49
50type oracleResult struct {
51 K string `json:"k"`
52 S string `json:"s"`
53 J string `json:"j"`
54 // U holds the UTF-16 code units of the string form when it contains a
55 // lone surrogate and S is therefore empty.
56 U []uint16 `json:"u"`
57}
58
59var oracleFuncInfos = []FuncInfo{
60 {Name: "success", MinArgs: 0, MaxArgs: 0},
61 {Name: "always", MinArgs: 0, MaxArgs: 0},
62 {Name: "hashFiles", MinArgs: 1, MaxArgs: 255},
63}
64
65var oracleFuncs = map[string]Func{
66 "success": func(context.Context, []Value) (Value, error) { return Boolean(true), nil },
67 "always": func(context.Context, []Value) (Value, error) { return Boolean(true), nil },
68 "hashFiles": func(_ context.Context, args []Value) (Value, error) {
69 s := "hash:"
70 for i, a := range args {
71 if i > 0 {
72 s += "|"
73 }
74 s += CoerceString(a)
75 }
76 return String(s), nil
77 },
78}
79
80func kindName(v Value) string {
81 switch v.Kind() {
82 case KindNull:
83 return "null"
84 case KindBoolean:
85 return "boolean"
86 case KindNumber:
87 return "number"
88 case KindString:
89 return "string"
90 case KindArray:
91 return "array"
92 }
93 return "object"
94}
95
96func loadOracle(tb testing.TB) (*oracleFile, map[string]Value, []string) {
97 tb.Helper()
98 f, err := os.Open("testdata/oracle.json.gz")
99 if err != nil {
100 tb.Fatal(err)
101 }
102 defer func() { _ = f.Close() }()
103 zr, err := gzip.NewReader(f)
104 if err != nil {
105 tb.Fatal(err)
106 }
107 var of oracleFile
108 if err := json.NewDecoder(zr).Decode(&of); err != nil {
109 tb.Fatal(err)
110 }
111 ctx := map[string]Value{}
112 var names []string
113 for name, raw := range of.Contexts {
114 v, err := FromJSON(raw)
115 if err != nil {
116 tb.Fatalf("context %s: %v", name, err)
117 }
118 ctx[name] = v
119 names = append(names, name)
120 }
121 return &of, ctx, names
122}
123
124// oracleRun carries the state shared by the checks of one oracle case.
125type oracleRun struct {
126 t *testing.T
127 ec EvalContext
128 ctxNames []string
129 failures int
130 c oracleCase
131 // pending holds the mismatches of the current case until the case is
132 // settled: a mismatch with the JavaScript reference is excused when the
133 // C# runner, which is what runs on github.com, gives this package's answer.
134 pending []string
135}
136
137func (r *oracleRun) fail(format string, args ...any) {
138 r.pending = append(r.pending, fmt.Sprintf(format, args...))
139}
140
141// settle reports the pending mismatches of the current case, unless the
142// runner agrees with this package. It returns true when the case was excused.
143func (r *oracleRun) settle(rc runnerCase, ctxNames []string) (excused bool) {
144 if len(r.pending) == 0 {
145 return false
146 }
147 defer func() { r.pending = r.pending[:0] }()
148 if rc.E == r.c.E && sameAnswer(goAnswer(r.c.E, ctxNames, r.ec), rc) {
149 return true
150 }
151 for _, m := range r.pending {
152 r.failures++
153 if r.failures <= 40 {
154 e := r.c.E
155 if len(e) > 200 {
156 e = e[:200] + "..."
157 }
158 r.t.Errorf("%q: %s", e, m)
159 }
160 }
161 return false
162}
163
164// checkLex compares the tokens and reports whether the case lexes.
165func (r *oracleRun) checkLex() bool {
166 c := r.c
167 tokens, err := Lex(c.E)
168 if c.LexErr != nil {
169 switch {
170 case err == nil:
171 r.fail("lex: want error %q, got none", *c.LexErr)
172 case err.Error() != *c.LexErr:
173 r.fail("lex error: want %q, got %q", *c.LexErr, err.Error())
174 }
175 return false
176 }
177 if err != nil {
178 r.fail("lex: unexpected error %v", err)
179 return false
180 }
181 if len(tokens) != len(c.Tokens) {
182 r.fail("lex: %d tokens, want %d", len(tokens), len(c.Tokens))
183 return false
184 }
185 for i, tk := range tokens {
186 w := c.Tokens[i]
187 got := []any{float64(tk.Type), tk.Lexeme, float64(tk.Range.Start.Line), float64(tk.Range.Start.Column), float64(tk.Range.End.Line), float64(tk.Range.End.Column)}
188 for j := range got {
189 if got[j] != w[j] {
190 r.fail("token %d field %d: got %v, want %v", i, j, got[j], w[j])
191 }
192 }
193 var gv any
194 switch tk.Type {
195 case TokenNumber:
196 f := float64(tk.Value.(Number))
197 if math.IsNaN(f) {
198 gv = "NaN"
199 } else {
200 gv = jsNumberToString(f)
201 }
202 case TokenString:
203 gv = string(tk.Value.(String))
204 }
205 if gv != w[6] {
206 r.fail("token %d value: got %v, want %v", i, gv, w[6])
207 }
208 }
209 return true
210}
211
212// checkParse compares the parse outcome and returns the tree when the case
213// parses to something to evaluate.
214func (r *oracleRun) checkParse() Node {
215 c := r.c
216 node, err := Parse(c.E, r.ctxNames, oracleFuncInfos)
217 if c.ParseErr != nil {
218 var pe *Error
219 switch {
220 case err == nil:
221 r.fail("parse: want error %q, got none", *c.ParseErr)
222 case err.Error() != *c.ParseErr:
223 r.fail("parse error: want %q, got %q", *c.ParseErr, err.Error())
224 case !errors.As(err, &pe):
225 r.fail("parse error has type %T", err)
226 case c.Pos != nil && (pe.Pos.Line != c.Pos[0] || pe.Pos.Column != c.Pos[1]):
227 r.fail("parse error position: got %v, want %v", pe.Pos, c.Pos)
228 }
229 return nil
230 }
231 if err != nil {
232 r.fail("parse: unexpected error %v", err)
233 return nil
234 }
235 if c.Empty {
236 if node != nil {
237 r.fail("parse: want empty")
238 }
239 return nil
240 }
241 return node
242}
243
244func (r *oracleRun) checkEval(node Node) {
245 c := r.c
246 v, err := Evaluate(context.Background(), node, r.ec)
247 if c.EvalErr != nil {
248 switch {
249 case err == nil:
250 r.fail("eval: want error %q, got %s", *c.EvalErr, CoerceString(v))
251 case err.Error() != *c.EvalErr:
252 r.fail("eval error: want %q, got %q", *c.EvalErr, err.Error())
253 }
254 return
255 }
256 if err != nil {
257 r.fail("eval: unexpected error %v", err)
258 return
259 }
260 if k := kindName(v); k != c.R.K {
261 r.fail("result kind: got %s, want %s", k, c.R.K)
262 }
263 if c.R.U != nil {
264 if got := utf16Units(CoerceString(v)); !unitsEqual(got, c.R.U) {
265 r.fail("result string units: got %04x, want %04x", got, c.R.U)
266 }
267 } else if s := CoerceString(v); s != c.R.S {
268 r.fail("result string: got %q, want %q", s, c.R.S)
269 }
270 if j := ToJSON(v); j != c.R.J {
271 r.fail("result JSON: got %q, want %q", j, c.R.J)
272 }
273}
274
275func TestOracle(t *testing.T) {
276 of, ctxValues, ctxNames := loadOracle(t)
277 if len(of.Cases) < 30000 {
278 t.Fatalf("oracle has only %d cases", len(of.Cases))
279 }
280 ref := of.Reference
281 if !strings.HasPrefix(ref.Integrity, "sha512-") {
282 t.Errorf("oracle does not record the integrity of the reference package: %+v", ref)
283 }
284 t.Logf("reference: %s, integrity %s, generated with node %s (V8 %s, Unicode %s)", ref.Package, ref.Integrity, ref.Node, ref.V8, ref.Unicode)
285
286 run := &oracleRun{t: t, ctxNames: ctxNames, ec: EvalContext{Contexts: ctxValues, Funcs: oracleFuncs, MaxResultBytes: -1}}
287 rf := loadRunner(t)
288 if len(rf.Cases) != len(of.Cases) {
289 t.Fatalf("runner answers %d cases, oracle has %d", len(rf.Cases), len(of.Cases))
290 }
291 var counts struct{ lexErr, parseErr, evalErr, ok, empty int }
292 excused := 0
293 for i, c := range of.Cases {
294 run.c = c
295 switch {
296 case c.LexErr != nil:
297 counts.lexErr++
298 case c.ParseErr != nil:
299 counts.parseErr++
300 case c.Empty:
301 counts.empty++
302 case c.EvalErr != nil:
303 counts.evalErr++
304 default:
305 counts.ok++
306 }
307 if run.checkLex() {
308 if node := run.checkParse(); node != nil {
309 run.checkEval(node)
310 }
311 }
312 if run.settle(rf.Cases[i], ctxNames) {
313 excused++
314 }
315 }
316 t.Logf("%d cases: %d lex errors, %d parse errors, %d empty, %d eval errors, %d values; %d differ from the JavaScript where the C# runner agrees with this package; %d failures",
317 len(of.Cases), counts.lexErr, counts.parseErr, counts.empty, counts.evalErr, counts.ok, excused, run.failures)
318 if run.failures > 0 {
319 t.Fatalf("%d oracle mismatches", run.failures)
320 }
321}