// SPDX-License-Identifier: MIT package expr import ( "compress/gzip" "context" "encoding/json" "errors" "fmt" "math" "os" "strings" "testing" ) // The oracle file holds what the reference implementation (GitHub's // @actions/expressions 0.3.61, run under Node) does on about 33 000 // expressions: hand-picked edge cases and a seeded random walk over the // grammar. It is generated by gen/oracle.mjs. TestOracle replays it and // demands identical tokens, error messages and positions, result kinds, // string coercions and toJSON output. type oracleFile struct { Reference oracleReference `json:"reference"` Contexts map[string]json.RawMessage `json:"contexts"` Cases []oracleCase `json:"cases"` } // oracleReference records where the oracle comes from. type oracleReference struct { Package string `json:"package"` Integrity string `json:"integrity"` Node string `json:"node"` V8 string `json:"v8"` Unicode string `json:"unicode"` } type oracleCase struct { E string `json:"e"` Tokens [][]any `json:"tokens"` LexErr *string `json:"lexErr"` ParseErr *string `json:"parseErr"` Pos []int `json:"pos"` Empty bool `json:"empty"` EvalErr *string `json:"evalErr"` R *oracleResult `json:"r"` } type oracleResult struct { K string `json:"k"` S string `json:"s"` J string `json:"j"` // U holds the UTF-16 code units of the string form when it contains a // lone surrogate and S is therefore empty. U []uint16 `json:"u"` } var oracleFuncInfos = []FuncInfo{ {Name: "success", MinArgs: 0, MaxArgs: 0}, {Name: "always", MinArgs: 0, MaxArgs: 0}, {Name: "hashFiles", MinArgs: 1, MaxArgs: 255}, } var oracleFuncs = map[string]Func{ "success": func(context.Context, []Value) (Value, error) { return Boolean(true), nil }, "always": func(context.Context, []Value) (Value, error) { return Boolean(true), nil }, "hashFiles": func(_ context.Context, args []Value) (Value, error) { s := "hash:" for i, a := range args { if i > 0 { s += "|" } s += CoerceString(a) } return String(s), nil }, } func kindName(v Value) string { switch v.Kind() { case KindNull: return "null" case KindBoolean: return "boolean" case KindNumber: return "number" case KindString: return "string" case KindArray: return "array" } return "object" } func loadOracle(tb testing.TB) (*oracleFile, map[string]Value, []string) { tb.Helper() f, err := os.Open("testdata/oracle.json.gz") if err != nil { tb.Fatal(err) } defer func() { _ = f.Close() }() zr, err := gzip.NewReader(f) if err != nil { tb.Fatal(err) } var of oracleFile if err := json.NewDecoder(zr).Decode(&of); err != nil { tb.Fatal(err) } ctx := map[string]Value{} var names []string for name, raw := range of.Contexts { v, err := FromJSON(raw) if err != nil { tb.Fatalf("context %s: %v", name, err) } ctx[name] = v names = append(names, name) } return &of, ctx, names } // oracleRun carries the state shared by the checks of one oracle case. type oracleRun struct { t *testing.T ec EvalContext ctxNames []string failures int c oracleCase // pending holds the mismatches of the current case until the case is // settled: a mismatch with the JavaScript reference is excused when the // C# runner, which is what runs on github.com, gives this package's answer. pending []string } func (r *oracleRun) fail(format string, args ...any) { r.pending = append(r.pending, fmt.Sprintf(format, args...)) } // settle reports the pending mismatches of the current case, unless the // runner agrees with this package. It returns true when the case was excused. func (r *oracleRun) settle(rc runnerCase, ctxNames []string) (excused bool) { if len(r.pending) == 0 { return false } defer func() { r.pending = r.pending[:0] }() if rc.E == r.c.E && sameAnswer(goAnswer(r.c.E, ctxNames, r.ec), rc) { return true } for _, m := range r.pending { r.failures++ if r.failures <= 40 { e := r.c.E if len(e) > 200 { e = e[:200] + "..." } r.t.Errorf("%q: %s", e, m) } } return false } // checkLex compares the tokens and reports whether the case lexes. func (r *oracleRun) checkLex() bool { c := r.c tokens, err := Lex(c.E) if c.LexErr != nil { switch { case err == nil: r.fail("lex: want error %q, got none", *c.LexErr) case err.Error() != *c.LexErr: r.fail("lex error: want %q, got %q", *c.LexErr, err.Error()) } return false } if err != nil { r.fail("lex: unexpected error %v", err) return false } if len(tokens) != len(c.Tokens) { r.fail("lex: %d tokens, want %d", len(tokens), len(c.Tokens)) return false } for i, tk := range tokens { w := c.Tokens[i] got := []any{float64(tk.Type), tk.Lexeme, float64(tk.Range.Start.Line), float64(tk.Range.Start.Column), float64(tk.Range.End.Line), float64(tk.Range.End.Column)} for j := range got { if got[j] != w[j] { r.fail("token %d field %d: got %v, want %v", i, j, got[j], w[j]) } } var gv any switch tk.Type { case TokenNumber: f := float64(tk.Value.(Number)) if math.IsNaN(f) { gv = "NaN" } else { gv = jsNumberToString(f) } case TokenString: gv = string(tk.Value.(String)) } if gv != w[6] { r.fail("token %d value: got %v, want %v", i, gv, w[6]) } } return true } // checkParse compares the parse outcome and returns the tree when the case // parses to something to evaluate. func (r *oracleRun) checkParse() Node { c := r.c node, err := Parse(c.E, r.ctxNames, oracleFuncInfos) if c.ParseErr != nil { var pe *Error switch { case err == nil: r.fail("parse: want error %q, got none", *c.ParseErr) case err.Error() != *c.ParseErr: r.fail("parse error: want %q, got %q", *c.ParseErr, err.Error()) case !errors.As(err, &pe): r.fail("parse error has type %T", err) case c.Pos != nil && (pe.Pos.Line != c.Pos[0] || pe.Pos.Column != c.Pos[1]): r.fail("parse error position: got %v, want %v", pe.Pos, c.Pos) } return nil } if err != nil { r.fail("parse: unexpected error %v", err) return nil } if c.Empty { if node != nil { r.fail("parse: want empty") } return nil } return node } func (r *oracleRun) checkEval(node Node) { c := r.c v, err := Evaluate(context.Background(), node, r.ec) if c.EvalErr != nil { switch { case err == nil: r.fail("eval: want error %q, got %s", *c.EvalErr, CoerceString(v)) case err.Error() != *c.EvalErr: r.fail("eval error: want %q, got %q", *c.EvalErr, err.Error()) } return } if err != nil { r.fail("eval: unexpected error %v", err) return } if k := kindName(v); k != c.R.K { r.fail("result kind: got %s, want %s", k, c.R.K) } if c.R.U != nil { if got := utf16Units(CoerceString(v)); !unitsEqual(got, c.R.U) { r.fail("result string units: got %04x, want %04x", got, c.R.U) } } else if s := CoerceString(v); s != c.R.S { r.fail("result string: got %q, want %q", s, c.R.S) } if j := ToJSON(v); j != c.R.J { r.fail("result JSON: got %q, want %q", j, c.R.J) } } func TestOracle(t *testing.T) { of, ctxValues, ctxNames := loadOracle(t) if len(of.Cases) < 30000 { t.Fatalf("oracle has only %d cases", len(of.Cases)) } ref := of.Reference if !strings.HasPrefix(ref.Integrity, "sha512-") { t.Errorf("oracle does not record the integrity of the reference package: %+v", ref) } t.Logf("reference: %s, integrity %s, generated with node %s (V8 %s, Unicode %s)", ref.Package, ref.Integrity, ref.Node, ref.V8, ref.Unicode) run := &oracleRun{t: t, ctxNames: ctxNames, ec: EvalContext{Contexts: ctxValues, Funcs: oracleFuncs, MaxResultBytes: -1}} rf := loadRunner(t) if len(rf.Cases) != len(of.Cases) { t.Fatalf("runner answers %d cases, oracle has %d", len(rf.Cases), len(of.Cases)) } var counts struct{ lexErr, parseErr, evalErr, ok, empty int } excused := 0 for i, c := range of.Cases { run.c = c switch { case c.LexErr != nil: counts.lexErr++ case c.ParseErr != nil: counts.parseErr++ case c.Empty: counts.empty++ case c.EvalErr != nil: counts.evalErr++ default: counts.ok++ } if run.checkLex() { if node := run.checkParse(); node != nil { run.checkEval(node) } } if run.settle(rf.Cases[i], ctxNames) { excused++ } } t.Logf("%d cases: %d lex errors, %d parse errors, %d empty, %d eval errors, %d values; %d differ from the JavaScript where the C# runner agrees with this package; %d failures", len(of.Cases), counts.lexErr, counts.parseErr, counts.empty, counts.evalErr, counts.ok, excused, run.failures) if run.failures > 0 { t.Fatalf("%d oracle mismatches", run.failures) } }