test(expr): excuse fromJSON error text differences only when both sides are reader failures (GO-004)

commit: 74a3afb1d052a6391a4234e5d66e46a0769ef4b4
author: brice · 10d ago · 2026-10-12 18:30 UTC
committer: brice · 4d ago · 2026-10-18 14:59 UTC
parent: c13f270
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

3 files · +61 -23

M expr/gen/runner/answers.mjs +0 -1


@@ -28,7 +28,6 @@ const git = (...a) => execFileSync("git", ["-C", runnerClone, ...a]).toString().
28	28		const dotnet = execFileSync("dotnet", ["--list-runtimes"]).toString().split("\n").find(l => l.includes("Microsoft.NETCore.App 9")) ?? "";
29	29		const cases = out.map((l, i) => {
30	30		const r = JSON.parse(l);
31		-	if (r.err) delete r.err.x; // .NET exception type name
32	31		return { e: oracle.cases[i].e, ...r };
33	32		});
34	33		process.stdout.write(JSON.stringify({


@@ -28,7 +28,6 @@ const git = (...a) => execFileSync("git", ["-C", runnerClone, ...a]).toString().
28		const dotnet = execFileSync("dotnet", ["--list-runtimes"]).toString().split("\n").find(l => l.includes("Microsoft.NETCore.App 9")) ?? "";	28	const dotnet = execFileSync("dotnet", ["--list-runtimes"]).toString().split("\n").find(l => l.includes("Microsoft.NETCore.App 9")) ?? "";
29		const cases = out.map((l, i) => {	29	const cases = out.map((l, i) => {
30		const r = JSON.parse(l);	30	const r = JSON.parse(l);
31	-	if (r.err) delete r.err.x; // .NET exception type name
32		return { e: oracle.cases[i].e, ...r };	31	return { e: oracle.cases[i].e, ...r };
33		});	32	});
34		process.stdout.write(JSON.stringify({	33	process.stdout.write(JSON.stringify({

M expr/runner_oracle_test.go +61 -22


@@ -6,6 +6,7 @@ import (
6	6		"compress/gzip"
7	7		"context"
8	8		"encoding/json"
	9	+	"errors"
9	10		"os"
10	11		"strings"
11	12		"testing"

@@ -29,15 +30,20 @@ type runnerFile struct {
29	30		}
30	31
31	32		type runnerCase struct {
32		-	E string `json:"e"`
33		-	Empty bool `json:"empty"`
34		-	K string `json:"k"`
35		-	S string `json:"s"`
36		-	J string `json:"j"`
37		-	Err *struct {
38		-	T string `json:"t"`
39		-	M string `json:"m"`
40		-	} `json:"err"`
	33	+	E string `json:"e"`
	34	+	Empty bool `json:"empty"`
	35	+	K string `json:"k"`
	36	+	S string `json:"s"`
	37	+	J string `json:"j"`
	38	+	Err *runnerErr `json:"err"`
	39	+	}
	40	+
	41	+	// runnerErr is an error of the runner: the phase, the message, and the name of
	42	+	// the .NET exception class ("JsonReaderException" for a fromJSON failure).
	43	+	type runnerErr struct {
	44	+	T string `json:"t"`
	45	+	M string `json:"m"`
	46	+	X string `json:"x"`
41	47		}
42	48
43	49		func loadRunner(tb testing.TB) *runnerFile {

@@ -83,10 +89,7 @@ func goAnswer(e string, ctxNames []string, ec EvalContext) runnerCase {
83	89		rc := runnerCase{E: e}
84	90		node, err := Parse(e, ctxNames, oracleFuncInfos)
85	91		if err != nil {
86		-	rc.Err = &struct {
87		-	T string `json:"t"`
88		-	M string `json:"m"`
89		-	}{"parse", err.Error()}
	92	+	rc.Err = &runnerErr{T: "parse", M: err.Error()}
90	93		return rc
91	94		}
92	95		if node == nil {

@@ -95,10 +98,7 @@ func goAnswer(e string, ctxNames []string, ec EvalContext) runnerCase {
95	98		}
96	99		v, err := Evaluate(context.Background(), node, ec)
97	100		if err != nil {
98		-	rc.Err = &struct {
99		-	T string `json:"t"`
100		-	M string `json:"m"`
101		-	}{"eval", err.Error()}
	101	+	rc.Err = &runnerErr{T: "eval", M: err.Error(), X: goErrorClass(err)}
102	102		return rc
103	103		}
104	104		rc.K = v.Kind().String()

@@ -107,11 +107,23 @@ func goAnswer(e string, ctxNames []string, ec EvalContext) runnerCase {
107	107		return rc
108	108		}
109	109
	110	+	// goErrorClass names the kind of an evaluation error of this package in the
	111	+	// terms of runnerErr.X: "JsonReaderException" for the failures of fromJSON.
	112	+	func goErrorClass(err error) string {
	113	+	var ee *EvalError
	114	+	if errors.As(err, &ee) && (ee.Message == "empty input" \|\| ee.Message == "Error parsing JSON when evaluating fromJson") {
	115	+	return "JsonReaderException"
	116	+	}
	117	+	return ""
	118	+	}
	119	+
110	120		// sameAnswer reports whether a Go answer matches the runner's. Errors match
111	121		// when they are of the same phase and, for parse errors, name the same kind of
112		-	// problem (the text before the colon); the runner appends the position and
113		-	// names the parenthesis rather than the function in arity errors, and its
114		-	// fromJSON errors are Json.NET messages. Other evaluation errors must be equal.
	122	+	// problem (the text before the colon: the runner appends the position, and its
	123	+	// arity errors name the parenthesis). Evaluation errors must have the same
	124	+	// message, except a fromJSON failure, whose runner message is Json.NET's: it
	125	+	// matches only a fromJSON failure of this package (both are JSON reader
	126	+	// failures, never just "the expression mentions fromJSON").
115	127		func sameAnswer(g, r runnerCase) bool {
116	128		if (g.Err == nil) != (r.Err == nil) {
117	129		return false

@@ -126,8 +138,8 @@ func sameAnswer(g, r runnerCase) bool {
126	138		desc := func(m string) string { d, _, _ := strings.Cut(m, ":"); return d }
127	139		return desc(g.Err.M) == desc(r.Err.M)
128	140		}
129		-	if strings.Contains(strings.ToLower(r.E), "fromjson") {
130		-	return true
	141	+	if r.Err.X == "JsonReaderException" \|\| g.Err.X == "JsonReaderException" {
	142	+	return r.Err.X == g.Err.X
131	143		}
132	144		return g.Err.M == r.Err.M
133	145		}

@@ -138,3 +150,30 @@ func clip(s string) string {
138	150		}
139	151		return s
140	152		}
	153	+
	154	+	// TestSameAnswerFromJSON: an error is excused as "a fromJSON failure" only
	155	+	// when both sides are one; an expression that merely mentions fromJSON does not
	156	+	// get a free pass.
	157	+	func TestSameAnswerFromJSON(t *testing.T) {
	158	+	ev := func(m, x string) runnerCase { return runnerCase{E: "fromJSON('x')", Err: &runnerErr{T: "eval", M: m, X: x}} }
	159	+	reader := ev("Error parsing JSON when evaluating fromJson", "JsonReaderException")
	160	+	runnerReader := ev("Unexpected character encountered while parsing value: x. Path '', line 0, position 0.", "JsonReaderException")
	161	+	format := ev("The following format string is invalid: {", "FormatException")
	162	+	value := runnerCase{E: "fromJSON('1')", K: "Number", S: "1", J: "1"}
	163	+	for _, tc := range []struct {
	164	+	name string
	165	+	g, r runnerCase
	166	+	want bool
	167	+	}{
	168	+	{"both reader failures", reader, runnerReader, true},
	169	+	{"go fails, runner succeeds", reader, value, false},
	170	+	{"runner fails, go succeeds", value, runnerReader, false},
	171	+	{"go reader failure, runner format error", reader, format, false},
	172	+	{"runner reader failure, go format error", format, runnerReader, false},
	173	+	{"same non-reader error", format, format, true},
	174	+	} {
	175	+	if got := sameAnswer(tc.g, tc.r); got != tc.want {
	176	+	t.Errorf("%s: sameAnswer = %v, want %v", tc.name, got, tc.want)
	177	+	}
	178	+	}
	179	+	}


@@ -6,6 +6,7 @@ import (
6		"compress/gzip"	6		"compress/gzip"
7		"context"	7		"context"
8		"encoding/json"	8		"encoding/json"
			9	+	"errors"
9		"os"	10		"os"
10		"strings"	11		"strings"
11		"testing"	12		"testing"

@@ -29,15 +30,20 @@ type runnerFile struct {
29		}	30		}
30			31
31		type runnerCase struct {	32		type runnerCase struct {
32	-	E string `json:"e"`	33	+	E string `json:"e"`
33	-	Empty bool `json:"empty"`	34	+	Empty bool `json:"empty"`
34	-	K string `json:"k"`	35	+	K string `json:"k"`
35	-	S string `json:"s"`	36	+	S string `json:"s"`
36	-	J string `json:"j"`	37	+	J string `json:"j"`
37	-	Err *struct {	38	+	Err *runnerErr `json:"err"`
38	-	T string `json:"t"`	39	+	}
39	-	M string `json:"m"`	40	+
40	-	} `json:"err"`	41	+	// runnerErr is an error of the runner: the phase, the message, and the name of
			42	+	// the .NET exception class ("JsonReaderException" for a fromJSON failure).
			43	+	type runnerErr struct {
			44	+	T string `json:"t"`
			45	+	M string `json:"m"`
			46	+	X string `json:"x"`
41		}	47		}
42			48
43		func loadRunner(tb testing.TB) *runnerFile {	49		func loadRunner(tb testing.TB) *runnerFile {

@@ -83,10 +89,7 @@ func goAnswer(e string, ctxNames []string, ec EvalContext) runnerCase {
83		rc := runnerCase{E: e}	89		rc := runnerCase{E: e}
84		node, err := Parse(e, ctxNames, oracleFuncInfos)	90		node, err := Parse(e, ctxNames, oracleFuncInfos)
85		if err != nil {	91		if err != nil {
86	-	rc.Err = &struct {	92	+	rc.Err = &runnerErr{T: "parse", M: err.Error()}
87	-	T string `json:"t"`
88	-	M string `json:"m"`
89	-	}{"parse", err.Error()}
90		return rc	93		return rc
91		}	94		}
92		if node == nil {	95		if node == nil {

@@ -95,10 +98,7 @@ func goAnswer(e string, ctxNames []string, ec EvalContext) runnerCase {
95		}	98		}
96		v, err := Evaluate(context.Background(), node, ec)	99		v, err := Evaluate(context.Background(), node, ec)
97		if err != nil {	100		if err != nil {
98	-	rc.Err = &struct {	101	+	rc.Err = &runnerErr{T: "eval", M: err.Error(), X: goErrorClass(err)}
99	-	T string `json:"t"`
100	-	M string `json:"m"`
101	-	}{"eval", err.Error()}
102		return rc	102		return rc
103		}	103		}
104		rc.K = v.Kind().String()	104		rc.K = v.Kind().String()

@@ -107,11 +107,23 @@ func goAnswer(e string, ctxNames []string, ec EvalContext) runnerCase {
107		return rc	107		return rc
108		}	108		}
109			109
			110	+	// goErrorClass names the kind of an evaluation error of this package in the
			111	+	// terms of runnerErr.X: "JsonReaderException" for the failures of fromJSON.
			112	+	func goErrorClass(err error) string {
			113	+	var ee *EvalError
			114	+	if errors.As(err, &ee) && (ee.Message == "empty input" \|\| ee.Message == "Error parsing JSON when evaluating fromJson") {
			115	+	return "JsonReaderException"
			116	+	}
			117	+	return ""
			118	+	}
			119	+
110		// sameAnswer reports whether a Go answer matches the runner's. Errors match	120		// sameAnswer reports whether a Go answer matches the runner's. Errors match
111		// when they are of the same phase and, for parse errors, name the same kind of	121		// when they are of the same phase and, for parse errors, name the same kind of
112	-	// problem (the text before the colon); the runner appends the position and	122	+	// problem (the text before the colon: the runner appends the position, and its
113	-	// names the parenthesis rather than the function in arity errors, and its	123	+	// arity errors name the parenthesis). Evaluation errors must have the same
114	-	// fromJSON errors are Json.NET messages. Other evaluation errors must be equal.	124	+	// message, except a fromJSON failure, whose runner message is Json.NET's: it
			125	+	// matches only a fromJSON failure of this package (both are JSON reader
			126	+	// failures, never just "the expression mentions fromJSON").
115		func sameAnswer(g, r runnerCase) bool {	127		func sameAnswer(g, r runnerCase) bool {
116		if (g.Err == nil) != (r.Err == nil) {	128		if (g.Err == nil) != (r.Err == nil) {
117		return false	129		return false

@@ -126,8 +138,8 @@ func sameAnswer(g, r runnerCase) bool {
126		desc := func(m string) string { d, _, _ := strings.Cut(m, ":"); return d }	138		desc := func(m string) string { d, _, _ := strings.Cut(m, ":"); return d }
127		return desc(g.Err.M) == desc(r.Err.M)	139		return desc(g.Err.M) == desc(r.Err.M)
128		}	140		}
129	-	if strings.Contains(strings.ToLower(r.E), "fromjson") {	141	+	if r.Err.X == "JsonReaderException" \|\| g.Err.X == "JsonReaderException" {
130	-	return true	142	+	return r.Err.X == g.Err.X
131		}	143		}
132		return g.Err.M == r.Err.M	144		return g.Err.M == r.Err.M
133		}	145		}

@@ -138,3 +150,30 @@ func clip(s string) string {
138		}	150		}
139		return s	151		return s
140		}	152		}
			153	+
			154	+	// TestSameAnswerFromJSON: an error is excused as "a fromJSON failure" only
			155	+	// when both sides are one; an expression that merely mentions fromJSON does not
			156	+	// get a free pass.
			157	+	func TestSameAnswerFromJSON(t *testing.T) {
			158	+	ev := func(m, x string) runnerCase { return runnerCase{E: "fromJSON('x')", Err: &runnerErr{T: "eval", M: m, X: x}} }
			159	+	reader := ev("Error parsing JSON when evaluating fromJson", "JsonReaderException")
			160	+	runnerReader := ev("Unexpected character encountered while parsing value: x. Path '', line 0, position 0.", "JsonReaderException")
			161	+	format := ev("The following format string is invalid: {", "FormatException")
			162	+	value := runnerCase{E: "fromJSON('1')", K: "Number", S: "1", J: "1"}
			163	+	for _, tc := range []struct {
			164	+	name string
			165	+	g, r runnerCase
			166	+	want bool
			167	+	}{
			168	+	{"both reader failures", reader, runnerReader, true},
			169	+	{"go fails, runner succeeds", reader, value, false},
			170	+	{"runner fails, go succeeds", value, runnerReader, false},
			171	+	{"go reader failure, runner format error", reader, format, false},
			172	+	{"runner reader failure, go format error", format, runnerReader, false},
			173	+	{"same non-reader error", format, format, true},
			174	+	} {
			175	+	if got := sameAnswer(tc.g, tc.r); got != tc.want {
			176	+	t.Errorf("%s: sameAnswer = %v, want %v", tc.name, got, tc.want)
			177	+	}
			178	+	}
			179	+	}

M expr/testdata/runner.json.gz +0 -0 binary file: no text diff

binary file: no text diff