fix(workflow): report matrix limit and exclude errors like GitHub, omit null and empty values from job names

commit: 2da867ffc7b2e57a51aa17f76be38638d3d490c6
author: brice · 10d ago · 2026-10-12 19:16 UTC
committer: brice · 4d ago · 2026-10-18 14:59 UTC
parent: ccaa3bb
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

3 files · +176 -52

M workflow/matrix.go +81 -34


@@ -34,8 +34,8 @@ type MatrixCombo struct {
34	34		// they were first added.
35	35		Values *expr.Dictionary
36	36		// Segments are the scalar values of Values, in order, shown in the name
37		-	// of the job: "build (ubuntu-latest, 14)". Arrays and mappings are left
38		-	// out; null shows as an empty segment.
	37	+	// of the job: "build (ubuntu-latest, 14)". Arrays, mappings, null and the
	38	+	// empty string are left out, like GitHub does (matrix-edge-values).
39	39		Segments []string
40	40		}
41	41

@@ -62,23 +62,23 @@ func (c MatrixCombo) DisplayName(base string) string {
62	62		// mentions no vector key matches all), without overwriting the original
63	63		// values; an entry that matches no combination becomes a combination of
64	64		// its own;
65		-	// - more than 256 jobs is an error. A strategy without a matrix gives a
66		-	// single combination.
	65	+	// - more than 256 jobs is a [*ConfigurationLimitError]. A strategy without a
	66	+	// matrix gives a single combination.
67	67		//
68	68		// The work is bounded: the product of the vectors may not exceed
69	69		// [MaxMatrixProduct], include and exclude may have [MaxMatrixEntries] entries
70	70		// each, and ctx is checked every few thousand combinations.
71	71		//
72		-	// TODO(verify with recording): the documented semantics of include and
73		-	// exclude are implemented as documented, but the following are assumed: the
74		-	// order of the cross product (first key slowest), the display name listing
75		-	// every scalar value including those added by include (see
76		-	// [MatrixCombo.DisplayName]) and showing null as an empty segment, the
77		-	// comparison of values (same kind, strings compared exactly, mappings by
78		-	// containment), an include key matching a vector key case-insensitively, the
79		-	// error when every combination is excluded, the limits above, and the
80		-	// wording of all errors except the unknown exclude key, which is GitHub's
81		-	// (also to verify).
	72	+	// Recorded on github.com (scenarios matrix-edge-values and matrix-over-256):
	73	+	// the unknown exclude key error, reported once per offending entry, the 256
	74	+	// limit with the number of configurations produced, and the omission of null
	75	+	// and ” from the default name. Still assumed, from the documented semantics
	76	+	// of include and exclude: the order of the cross product (first key slowest),
	77	+	// the display name listing every scalar value including those added by
	78	+	// include (see [MatrixCombo.DisplayName]), the comparison of values (same
	79	+	// kind, strings compared exactly, mappings by containment), an include key
	80	+	// matching a vector key case-insensitively, the error when every combination
	81	+	// is excluded, the other limits above, and the wording of the other errors.
82	82		func ExpandMatrix(ctx context.Context, strategy expr.Value) ([]MatrixCombo, error) {
83	83		s, err := ParseStrategy(ctx, strategy)
84	84		if err != nil {

@@ -121,7 +121,7 @@ func buildMatrix(ctx context.Context, m expr.Value) ([]MatrixCombo, bool, error)
121	121		return nil, false, fmt.Errorf("matrix vector '%s' does not contain any values", p.Key)
122	122		}
123	123		if arr.Len() > MaxMatrixProduct {
124		-	return nil, false, errTooBig
	124	+	return nil, false, errProductTooBig
125	125		}
126	126		vecs = append(vecs, vector{key: p.Key, items: arr.Items()})
127	127		}

@@ -153,7 +153,30 @@ type message string
153	153
154	154		func (m message) Error() string { return string(m) }
155	155
156		-	var errTooBig = fmt.Errorf("matrix size limit of %d exceeded", MaxMatrixJobs)
	156	+	// errProductTooBig is returned when the cross product of the vectors is too
	157	+	// large to be enumerated. GitHub would count the configurations left after
	158	+	// the exclusions; this limit keeps the work bounded, see [MaxMatrixProduct].
	159	+	var errProductTooBig = fmt.Errorf("matrix too large: the cross product of the vectors exceeds %d combinations", MaxMatrixProduct)
	160	+
	161	+	// ConfigurationLimitError is returned when a matrix produces more than
	162	+	// [MaxMatrixJobs] configurations. Its text is GitHub's (recorded by
	163	+	// matrix-over-256): "Strategy for job 'too-many' produced 272 configurations
	164	+	// which exceeds the maximum of 256 configurations". Job is empty when the
	165	+	// matrix was expanded without a job, see [Job.EvaluateStrategy].
	166	+	type ConfigurationLimitError struct {
	167	+	// Job is the id of the job.
	168	+	Job string
	169	+	// Count is the number of configurations the matrix produces.
	170	+	Count int
	171	+	}
	172	+
	173	+	func (e *ConfigurationLimitError) Error() string {
	174	+	subject := "Strategy"
	175	+	if e.Job != "" {
	176	+	subject = fmt.Sprintf("Strategy for job '%s'", e.Job)
	177	+	}
	178	+	return fmt.Sprintf("%s produced %d configurations which exceeds the maximum of %d configurations", subject, e.Count, MaxMatrixJobs)
	179	+	}
157	180
158	181		type vector struct {
159	182		key string

@@ -188,19 +211,27 @@ func expand(ctx context.Context, vecs []vector, includes, excludes []*expr.Dicti
188	211		// Exclude entries become lists of conditions on item positions, so that
189	212		// testing a combination costs a few array reads.
190	213		var excl [][]condition
	214	+	var bad []error
191	215		for _, ex := range excludes {
192	216		var conds []condition
	217	+	unknown := false
193	218		for _, p := range ex.Pairs() {
194	219		vi, ok := index[strings.ToLower(p.Key)]
195	220		if !ok {
196		-	return nil, false, message(fmt.Sprintf("Matrix exclude key '%s' does not match any key within the matrix", p.Key))
	221	+	// One error per offending entry, for its first unknown key.
	222	+	bad = append(bad, message(fmt.Sprintf("Matrix exclude key '%s' does not match any key within the matrix", p.Key)))
	223	+	unknown = true
	224	+	break
197	225		}
198	226		conds = append(conds, compileCondition(vecs, vi, p.Value))
199	227		}
200		-	if len(conds) > 0 {
	228	+	if len(conds) > 0 && !unknown {
201	229		excl = append(excl, conds)
202	230		}
203	231		}
	232	+	if len(bad) > 0 {
	233	+	return nil, false, errors.Join(bad...)
	234	+	}
204	235		incl := make([]includeEntry, len(includes))
205	236		for n, inc := range includes {
206	237		for _, p := range inc.Pairs() {

@@ -212,14 +243,17 @@ func expand(ctx context.Context, vecs []vector, includes, excludes []*expr.Dicti
212	243		}
213	244		}
214	245
	246	+	// total counts every configuration, so that the error of a matrix over the
	247	+	// limit tells how many it produces; only the first MaxMatrixJobs are built.
215	248		var combos []MatrixCombo
	249	+	total := 0
216	250		matched := make([]bool, len(includes))
217	251		if len(vecs) > 0 {
218	252		product := 1
219	253		for _, v := range vecs {
220	254		product *= len(v.items)
221	255		if product > MaxMatrixProduct {
222		-	return nil, false, errTooBig
	256	+	return nil, false, errProductTooBig
223	257		}
224	258		}
225	259		tuple := make([]int, len(vecs))

@@ -245,22 +279,28 @@ func expand(ctx context.Context, vecs []vector, includes, excludes []*expr.Dicti
245	279		continue next
246	280		}
247	281		}
248		-	b := &comboBuilder{vals: map[string]expr.Value{}}
249		-	for k, v := range vecs {
250		-	b.set(v.key, v.items[tuple[k]])
	282	+	total++
	283	+	build := total <= MaxMatrixJobs
	284	+	var b *comboBuilder
	285	+	if build {
	286	+	b = &comboBuilder{vals: map[string]expr.Value{}}
	287	+	for k, v := range vecs {
	288	+	b.set(v.key, v.items[tuple[k]])
	289	+	}
251	290		}
252	291		for n, inc := range incl {
253	292		if !allHold(inc.conds, tuple) {
254	293		continue
255	294		}
256	295		matched[n] = true
257		-	for _, p := range inc.extra {
258		-	b.set(p.Key, p.Value)
	296	+	if build {
	297	+	for _, p := range inc.extra {
	298	+	b.set(p.Key, p.Value)
	299	+	}
259	300		}
260	301		}
261		-	combos = append(combos, b.combo())
262		-	if len(combos) > MaxMatrixJobs {
263		-	return nil, false, errTooBig
	302	+	if build {
	303	+	combos = append(combos, b.combo())
264	304		}
265	305		}
266	306		}

@@ -268,14 +308,18 @@ func expand(ctx context.Context, vecs []vector, includes, excludes []*expr.Dicti
268	308		if matched[n] {
269	309		continue
270	310		}
	311	+	total++
	312	+	if total > MaxMatrixJobs {
	313	+	continue
	314	+	}
271	315		b := &comboBuilder{vals: map[string]expr.Value{}}
272	316		for _, p := range inc.Pairs() {
273	317		b.set(p.Key, p.Value)
274	318		}
275	319		combos = append(combos, b.combo())
276		-	if len(combos) > MaxMatrixJobs {
277		-	return nil, false, errTooBig
278		-	}
	320	+	}
	321	+	if total > MaxMatrixJobs {
	322	+	return nil, false, &ConfigurationLimitError{Count: total}
279	323		}
280	324		if len(combos) == 0 {
281	325		if len(vecs) == 0 && len(includes) == 0 && len(excludes) == 0 {

@@ -368,12 +412,15 @@ func valuesMatch(have, want expr.Value) bool {
368	412		return false
369	413		}
370	414
371		-	// segment is the text a matrix value adds to the name of a job; arrays and
372		-	// mappings add nothing.
	415	+	// segment is the text a matrix value adds to the name of a job; arrays,
	416	+	// mappings, null and the empty string add nothing (matrix-edge-values:
	417	+	// "default-name", "two-axes (x)"); numbers and booleans are shown as in an
	418	+	// expression ("1.5", "true").
373	419		func segment(v expr.Value) (string, bool) {
374	420		switch v.(type) {
375		-	case expr.Array, expr.Dictionary:
	421	+	case nil, expr.Null, expr.Array, expr.Dictionary:
376	422		return "", false
377	423		}
378		-	return expr.CoerceString(v), true
	424	+	s := expr.CoerceString(v)
	425	+	return s, s != ""
379	426		}

 	// they were first added.
 	Values *expr.Dictionary
 	// Segments are the scalar values of Values, in order, shown in the name
-	// of the job: "build (ubuntu-latest, 14)". Arrays and mappings are left
+	// of the job: "build (ubuntu-latest, 14)". Arrays, mappings, null and the
-	// out; null shows as an empty segment.
+	// empty string are left out, like GitHub does (matrix-edge-values).
 	Segments []string
 }
 //     mentions no vector key matches all), without overwriting the original
 //     values; an entry that matches no combination becomes a combination of
 //     its own;
-//   - more than 256 jobs is an error. A strategy without a matrix gives a
+//   - more than 256 jobs is a [*ConfigurationLimitError]. A strategy without a
-//     single combination.
+//     matrix gives a single combination.
 //
 // The work is bounded: the product of the vectors may not exceed
 // [MaxMatrixProduct], include and exclude may have [MaxMatrixEntries] entries
 // each, and ctx is checked every few thousand combinations.
 //
-// TODO(verify with recording): the documented semantics of include and
+// Recorded on github.com (scenarios matrix-edge-values and matrix-over-256):
-// exclude are implemented as documented, but the following are assumed: the
+// the unknown exclude key error, reported once per offending entry, the 256
-// order of the cross product (first key slowest), the display name listing
+// limit with the number of configurations produced, and the omission of null
-// every scalar value including those added by include (see
+// and ” from the default name. Still assumed, from the documented semantics
-// [MatrixCombo.DisplayName]) and showing null as an empty segment, the
+// of include and exclude: the order of the cross product (first key slowest),
-// comparison of values (same kind, strings compared exactly, mappings by
+// the display name listing every scalar value including those added by
-// containment), an include key matching a vector key case-insensitively, the
+// include (see [MatrixCombo.DisplayName]), the comparison of values (same
-// error when every combination is excluded, the limits above, and the
+// kind, strings compared exactly, mappings by containment), an include key
-// wording of all errors except the unknown exclude key, which is GitHub's
+// matching a vector key case-insensitively, the error when every combination
-// (also to verify).
+// is excluded, the other limits above, and the wording of the other errors.
 func ExpandMatrix(ctx context.Context, strategy expr.Value) ([]MatrixCombo, error) {
 	s, err := ParseStrategy(ctx, strategy)
 	if err != nil {
 				return nil, false, fmt.Errorf("matrix vector '%s' does not contain any values", p.Key)
 			}
 			if arr.Len() > MaxMatrixProduct {
-				return nil, false, errTooBig
+				return nil, false, errProductTooBig
 			}
 			vecs = append(vecs, vector{key: p.Key, items: arr.Items()})
 		}
 func (m message) Error() string { return string(m) }
-var errTooBig = fmt.Errorf("matrix size limit of %d exceeded", MaxMatrixJobs)
+// errProductTooBig is returned when the cross product of the vectors is too
+// large to be enumerated. GitHub would count the configurations left after
+// the exclusions; this limit keeps the work bounded, see [MaxMatrixProduct].
+var errProductTooBig = fmt.Errorf("matrix too large: the cross product of the vectors exceeds %d combinations", MaxMatrixProduct)
+// ConfigurationLimitError is returned when a matrix produces more than
+// [MaxMatrixJobs] configurations. Its text is GitHub's (recorded by
+// matrix-over-256): "Strategy for job 'too-many' produced 272 configurations
+// which exceeds the maximum of 256 configurations". Job is empty when the
+// matrix was expanded without a job, see [Job.EvaluateStrategy].
+type ConfigurationLimitError struct {
+	// Job is the id of the job.
+	Job string
+	// Count is the number of configurations the matrix produces.
+	Count int
+}
+func (e *ConfigurationLimitError) Error() string {
+	subject := "Strategy"
+	if e.Job != "" {
+		subject = fmt.Sprintf("Strategy for job '%s'", e.Job)
+	}
+	return fmt.Sprintf("%s produced %d configurations which exceeds the maximum of %d configurations", subject, e.Count, MaxMatrixJobs)
+}
 type vector struct {
 	key   string
 	// Exclude entries become lists of conditions on item positions, so that
 	// testing a combination costs a few array reads.
 	var excl [][]condition
+	var bad []error
 	for _, ex := range excludes {
 		var conds []condition
+		unknown := false
 		for _, p := range ex.Pairs() {
 			vi, ok := index[strings.ToLower(p.Key)]
 			if !ok {
-				return nil, false, message(fmt.Sprintf("Matrix exclude key '%s' does not match any key within the matrix", p.Key))
+				// One error per offending entry, for its first unknown key.
+				bad = append(bad, message(fmt.Sprintf("Matrix exclude key '%s' does not match any key within the matrix", p.Key)))
+				unknown = true
+				break
 			}
 			conds = append(conds, compileCondition(vecs, vi, p.Value))
 		}
-		if len(conds) > 0 {
+		if len(conds) > 0 && !unknown {
 			excl = append(excl, conds)
 		}
 	}
+	if len(bad) > 0 {
+		return nil, false, errors.Join(bad...)
+	}
 	incl := make([]includeEntry, len(includes))
 	for n, inc := range includes {
 		for _, p := range inc.Pairs() {
 		}
 	}
+	// total counts every configuration, so that the error of a matrix over the
+	// limit tells how many it produces; only the first MaxMatrixJobs are built.
 	var combos []MatrixCombo
+	total := 0
 	matched := make([]bool, len(includes))
 	if len(vecs) > 0 {
 		product := 1
 		for _, v := range vecs {
 			product *= len(v.items)
 			if product > MaxMatrixProduct {
-				return nil, false, errTooBig
+				return nil, false, errProductTooBig
 			}
 		}
 		tuple := make([]int, len(vecs))
 					continue next
 				}
 			}
-			b := &comboBuilder{vals: map[string]expr.Value{}}
+			total++
-			for k, v := range vecs {
+			build := total <= MaxMatrixJobs
-				b.set(v.key, v.items[tuple[k]])
+			var b *comboBuilder
+			if build {
+				b = &comboBuilder{vals: map[string]expr.Value{}}
+				for k, v := range vecs {
+					b.set(v.key, v.items[tuple[k]])
+				}
 			}
 			for n, inc := range incl {
 				if !allHold(inc.conds, tuple) {
 					continue
 				}
 				matched[n] = true
-				for _, p := range inc.extra {
+				if build {
-					b.set(p.Key, p.Value)
+					for _, p := range inc.extra {
+						b.set(p.Key, p.Value)
+					}
 				}
 			}
-			combos = append(combos, b.combo())
+			if build {
-			if len(combos) > MaxMatrixJobs {
+				combos = append(combos, b.combo())
-				return nil, false, errTooBig
 			}
 		}
 	}
 		if matched[n] {
 			continue
 		}
+		total++
+		if total > MaxMatrixJobs {
+			continue
+		}
 		b := &comboBuilder{vals: map[string]expr.Value{}}
 		for _, p := range inc.Pairs() {
 			b.set(p.Key, p.Value)
 		}
 		combos = append(combos, b.combo())
-		if len(combos) > MaxMatrixJobs {
+	}
-			return nil, false, errTooBig
+	if total > MaxMatrixJobs {
-		}
+		return nil, false, &ConfigurationLimitError{Count: total}
 	}
 	if len(combos) == 0 {
 		if len(vecs) == 0 && len(includes) == 0 && len(excludes) == 0 {
 	return false
 }
-// segment is the text a matrix value adds to the name of a job; arrays and
+// segment is the text a matrix value adds to the name of a job; arrays,
-// mappings add nothing.
+// mappings, null and the empty string add nothing (matrix-edge-values:
+// "default-name", "two-axes (x)"); numbers and booleans are shown as in an
+// expression ("1.5", "true").
 func segment(v expr.Value) (string, bool) {
 	switch v.(type) {
-	case *expr.Array, *expr.Dictionary:
+	case nil, expr.Null, *expr.Array, *expr.Dictionary:
 		return "", false
 	}
-	return expr.CoerceString(v), true
+	s := expr.CoerceString(v)
+	return s, s != ""
 }

M workflow/matrix_test.go +84 -13


@@ -140,7 +140,7 @@ func TestExpandMatrix(t *testing.T) {
140	140		{
141	141		"booleans, null and numbers in names",
142	142		`{"matrix": {"a": [true, null, 1.5]}}`,
143		-	[]string{`a=true \| job (true)`, `a=null \| job ()`, `a=1.5 \| job (1.5)`},
	143	+	[]string{`a=true \| job (true)`, `a=null \| job`, `a=1.5 \| job (1.5)`},
144	144		},
145	145		{
146	146		"objects and arrays are not in names",

@@ -212,7 +212,7 @@ func TestMatrixLimit(t *testing.T) {
212	212		t.Errorf("256 jobs: %d, %v", len(c), err)
213	213		}
214	214		_, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(16)+`, "b": `+vec(16)+`, "include": [{"a": 99}]}}`))
215		-	if err == nil \|\| !strings.Contains(err.Error(), "matrix size limit of 256 exceeded") {
	215	+	if err == nil \|\| !strings.Contains(err.Error(), "Strategy produced 257 configurations which exceeds the maximum of 256 configurations") {
216	216		t.Errorf("257 jobs: %v", err)
217	217		}
218	218		_, err = ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(17)+`, "b": `+vec(16)+`}}`))

@@ -290,7 +290,7 @@ func TestHostileMatrixFailsFast(t *testing.T) {
290	290		src := `{"matrix": {"a": ` + vec(100) + `, "b": ` + vec(100) + `, "c": ` + vec(100) + `, "exclude": ` + strings.ReplaceAll(entries(256), `"a": 1`, `"c": 5`) + `}}`
291	291		start := time.Now()
292	292		_, err := ExpandMatrix(ctxbg, jsonValue(t, src))
293		-	if err == nil \|\| !strings.Contains(err.Error(), "matrix size limit") {
	293	+	if err == nil \|\| !strings.Contains(err.Error(), "matrix too large") {
294	294		t.Errorf("err = %v", err)
295	295		}
296	296		if d := time.Since(start); d > time.Second {

@@ -321,11 +321,12 @@ func TestMatrixProductCap(t *testing.T) {
321	321		}
322	322		// 2^16 is allowed by the cap (and fails on the job limit), 2^16+1 is refused outright.
323	323		_, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(MaxMatrixProduct/2)+`, "b": `+vec(2)+`, "c": `+vec(1)+`}}`))
324		-	if err == nil \|\| !strings.Contains(err.Error(), "matrix size limit") {
	324	+	var limit *ConfigurationLimitError
	325	+	if !errors.As(err, &limit) \|\| limit.Count != MaxMatrixProduct {
325	326		t.Errorf("err = %v", err)
326	327		}
327	328		_, err = ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(MaxMatrixProduct+1)+`}}`))
328		-	if err == nil \|\| !strings.Contains(err.Error(), "matrix size limit") {
	329	+	if err == nil \|\| !strings.Contains(err.Error(), "matrix too large") {
329	330		t.Errorf("one huge vector: %v", err)
330	331		}
331	332		}

@@ -348,15 +349,85 @@ func TestMatrixContextIsHonoured(t *testing.T) {
348	349		}
349	350		}
350	351
351		-	func TestNullMatrixValueSegment(t *testing.T) {
352		-	// TODO(verify with recording): a null value shows as an empty segment,
353		-	// "job (, 1)"; GitHub may skip it.
354		-	combos, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": [null], "b": [1]}}`))
355		-	if err != nil {
356		-	t.Fatal(err)
	352	+	// TestRecordedMatrixNames follows the job names GitHub gave in the scenarios
	353	+	// matrix-edge-values (b-null-values.yml) and matrix-job-name.
	354	+	func TestRecordedMatrixNames(t *testing.T) {
	355	+	names := func(matrix string) []string {
	356	+	t.Helper()
	357	+	combos, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": `+matrix+`}`))
	358	+	if err != nil {
	359	+	t.Fatal(err)
	360	+	}
	361	+	var out []string
	362	+	for _, c := range combos {
	363	+	out = append(out, c.DisplayName("default-name"))
	364	+	}
	365	+	return out
	366	+	}
	367	+	// v: [1, null, '', true, 1.50, 'x y'] gave default-name, default-name,
	368	+	// default-name (1), default-name (1.5), default-name (true) and
	369	+	// default-name (x y): null and '' add nothing, 1.50 is 1.5.
	370	+	want := []string{"default-name (1)", "default-name", "default-name", "default-name (true)", "default-name (1.5)", "default-name (x y)"}
	371	+	if got := names(`{"v": [1, null, "", true, 1.50, "x y"]}`); !reflect.DeepEqual(got, want) {
	372	+	t.Errorf("one axis: got %q, want %q", got, want)
	373	+	}
	374	+	// a: [x, null], b: ['', y] gave "two-axes (x)", "two-axes (x, y)",
	375	+	// "two-axes (y)" and "two-axes".
	376	+	want = []string{"default-name (x)", "default-name (x, y)", "default-name", "default-name (y)"}
	377	+	if got := names(`{"a": ["x", null], "b": ["", "y"]}`); !reflect.DeepEqual(got, want) {
	378	+	t.Errorf("two axes: got %q, want %q", got, want)
	379	+	}
	380	+	// matrix-job-name: os [a, b] x n [1, 2] gave "default-name (a, 1)" ...
	381	+	want = []string{"default-name (a, 1)", "default-name (a, 2)", "default-name (b, 1)", "default-name (b, 2)"}
	382	+	if got := names(`{"os": ["a", "b"], "n": [1, 2]}`); !reflect.DeepEqual(got, want) {
	383	+	t.Errorf("matrix-job-name: got %q, want %q", got, want)
	384	+	}
	385	+	// matrix-value-types: objects are not listed ("types 2.5 false" has a
	386	+	// custom name, so only the default name is checked here).
	387	+	if got := names(`{"n": [1], "obj": [{"k": "v"}]}`); !reflect.DeepEqual(got, []string{"default-name (1)"}) {
	388	+	t.Errorf("object value: got %q", got)
357	389		}
358		-	if got := combos[0].DisplayName("job"); got != "job (, 1)" {
359		-	t.Errorf("name = %q", got)
	390	+	}
	391	+
	392	+	// TestRecordedExcludeUnknownKey is matrix-edge-values a-exclude-unknown-key.yml:
	393	+	// exclude: [{nokey: x}, {os: one, nokey: y}] made the workflow invalid.
	394	+	func TestRecordedExcludeUnknownKey(t *testing.T) {
	395	+	_, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"os": ["one", "two"], "exclude": [{"nokey": "x"}, {"os": "one", "nokey": "y"}]}}`))
	396	+	const one = "Matrix exclude key 'nokey' does not match any key within the matrix"
	397	+	if err == nil \|\| err.Error() != one+"\n"+one {
	398	+	t.Fatalf("err = %v, want the message once per offending entry", err)
	399	+	}
	400	+	// An entry that is fine does not add one.
	401	+	_, err = ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"os": ["one", "two"], "exclude": [{"os": "one"}, {"nokey": "x"}]}}`))
	402	+	if err == nil \|\| err.Error() != one {
	403	+	t.Errorf("err = %v, want %q", err, one)
	404	+	}
	405	+	}
	406	+
	407	+	// TestRecordedMatrixOver256 is the scenario matrix-over-256: 17 x 16 = 272
	408	+	// configurations in job too-many made the workflow invalid.
	409	+	func TestRecordedMatrixOver256(t *testing.T) {
	410	+	w, errs := Parse("ci.yml", []byte(`on: push
	411	+	jobs:
	412	+	too-many:
	413	+	runs-on: x
	414	+	strategy:
	415	+	matrix:
	416	+	a: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]
	417	+	b: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
	418	+	steps: [{run: a}]
	419	+	`))
	420	+	if len(errs) > 0 {
	421	+	t.Fatal(errs)
	422	+	}
	423	+	_, err := w.Job("too-many").EvaluateStrategy(ctxbg, template.Env{})
	424	+	const want = "Strategy for job 'too-many' produced 272 configurations which exceeds the maximum of 256 configurations"
	425	+	if err == nil \|\| err.Error() != want {
	426	+	t.Errorf("err = %v, want %q", err, want)
	427	+	}
	428	+	var limit *ConfigurationLimitError
	429	+	if !errors.As(err, &limit) \|\| limit.Job != "too-many" \|\| limit.Count != 272 {
	430	+	t.Errorf("err = %#v", err)
360	431		}
361	432		}
362	433


@@ -140,7 +140,7 @@ func TestExpandMatrix(t *testing.T) {
140		{	140		{
141		"booleans, null and numbers in names",	141		"booleans, null and numbers in names",
142		`{"matrix": {"a": [true, null, 1.5]}}`,	142		`{"matrix": {"a": [true, null, 1.5]}}`,
143	-	[]string{`a=true \| job (true)`, `a=null \| job ()`, `a=1.5 \| job (1.5)`},	143	+	[]string{`a=true \| job (true)`, `a=null \| job`, `a=1.5 \| job (1.5)`},
144		},	144		},
145		{	145		{
146		"objects and arrays are not in names",	146		"objects and arrays are not in names",

@@ -212,7 +212,7 @@ func TestMatrixLimit(t *testing.T) {
212		t.Errorf("256 jobs: %d, %v", len(c), err)	212		t.Errorf("256 jobs: %d, %v", len(c), err)
213		}	213		}
214		_, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(16)+`, "b": `+vec(16)+`, "include": [{"a": 99}]}}`))	214		_, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(16)+`, "b": `+vec(16)+`, "include": [{"a": 99}]}}`))
215	-	if err == nil \|\| !strings.Contains(err.Error(), "matrix size limit of 256 exceeded") {	215	+	if err == nil \|\| !strings.Contains(err.Error(), "Strategy produced 257 configurations which exceeds the maximum of 256 configurations") {
216		t.Errorf("257 jobs: %v", err)	216		t.Errorf("257 jobs: %v", err)
217		}	217		}
218		_, err = ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(17)+`, "b": `+vec(16)+`}}`))	218		_, err = ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(17)+`, "b": `+vec(16)+`}}`))

@@ -290,7 +290,7 @@ func TestHostileMatrixFailsFast(t *testing.T) {
290		src := `{"matrix": {"a": ` + vec(100) + `, "b": ` + vec(100) + `, "c": ` + vec(100) + `, "exclude": ` + strings.ReplaceAll(entries(256), `"a": 1`, `"c": 5`) + `}}`	290		src := `{"matrix": {"a": ` + vec(100) + `, "b": ` + vec(100) + `, "c": ` + vec(100) + `, "exclude": ` + strings.ReplaceAll(entries(256), `"a": 1`, `"c": 5`) + `}}`
291		start := time.Now()	291		start := time.Now()
292		_, err := ExpandMatrix(ctxbg, jsonValue(t, src))	292		_, err := ExpandMatrix(ctxbg, jsonValue(t, src))
293	-	if err == nil \|\| !strings.Contains(err.Error(), "matrix size limit") {	293	+	if err == nil \|\| !strings.Contains(err.Error(), "matrix too large") {
294		t.Errorf("err = %v", err)	294		t.Errorf("err = %v", err)
295		}	295		}
296		if d := time.Since(start); d > time.Second {	296		if d := time.Since(start); d > time.Second {

@@ -321,11 +321,12 @@ func TestMatrixProductCap(t *testing.T) {
321		}	321		}
322		// 2^16 is allowed by the cap (and fails on the job limit), 2^16+1 is refused outright.	322		// 2^16 is allowed by the cap (and fails on the job limit), 2^16+1 is refused outright.
323		_, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(MaxMatrixProduct/2)+`, "b": `+vec(2)+`, "c": `+vec(1)+`}}`))	323		_, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(MaxMatrixProduct/2)+`, "b": `+vec(2)+`, "c": `+vec(1)+`}}`))
324	-	if err == nil \|\| !strings.Contains(err.Error(), "matrix size limit") {	324	+	var limit *ConfigurationLimitError
			325	+	if !errors.As(err, &limit) \|\| limit.Count != MaxMatrixProduct {
325		t.Errorf("err = %v", err)	326		t.Errorf("err = %v", err)
326		}	327		}
327		_, err = ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(MaxMatrixProduct+1)+`}}`))	328		_, err = ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": `+vec(MaxMatrixProduct+1)+`}}`))
328	-	if err == nil \|\| !strings.Contains(err.Error(), "matrix size limit") {	329	+	if err == nil \|\| !strings.Contains(err.Error(), "matrix too large") {
329		t.Errorf("one huge vector: %v", err)	330		t.Errorf("one huge vector: %v", err)
330		}	331		}
331		}	332		}

@@ -348,15 +349,85 @@ func TestMatrixContextIsHonoured(t *testing.T) {
348		}	349		}
349		}	350		}
350			351
351	-	func TestNullMatrixValueSegment(t *testing.T) {	352	+	// TestRecordedMatrixNames follows the job names GitHub gave in the scenarios
352	-	// TODO(verify with recording): a null value shows as an empty segment,	353	+	// matrix-edge-values (b-null-values.yml) and matrix-job-name.
353	-	// "job (, 1)"; GitHub may skip it.	354	+	func TestRecordedMatrixNames(t *testing.T) {
354	-	combos, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"a": [null], "b": [1]}}`))	355	+	names := func(matrix string) []string {
355	-	if err != nil {	356	+	t.Helper()
356	-	t.Fatal(err)	357	+	combos, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": `+matrix+`}`))
			358	+	if err != nil {
			359	+	t.Fatal(err)
			360	+	}
			361	+	var out []string
			362	+	for _, c := range combos {
			363	+	out = append(out, c.DisplayName("default-name"))
			364	+	}
			365	+	return out
			366	+	}
			367	+	// v: [1, null, '', true, 1.50, 'x y'] gave default-name, default-name,
			368	+	// default-name (1), default-name (1.5), default-name (true) and
			369	+	// default-name (x y): null and '' add nothing, 1.50 is 1.5.
			370	+	want := []string{"default-name (1)", "default-name", "default-name", "default-name (true)", "default-name (1.5)", "default-name (x y)"}
			371	+	if got := names(`{"v": [1, null, "", true, 1.50, "x y"]}`); !reflect.DeepEqual(got, want) {
			372	+	t.Errorf("one axis: got %q, want %q", got, want)
			373	+	}
			374	+	// a: [x, null], b: ['', y] gave "two-axes (x)", "two-axes (x, y)",
			375	+	// "two-axes (y)" and "two-axes".
			376	+	want = []string{"default-name (x)", "default-name (x, y)", "default-name", "default-name (y)"}
			377	+	if got := names(`{"a": ["x", null], "b": ["", "y"]}`); !reflect.DeepEqual(got, want) {
			378	+	t.Errorf("two axes: got %q, want %q", got, want)
			379	+	}
			380	+	// matrix-job-name: os [a, b] x n [1, 2] gave "default-name (a, 1)" ...
			381	+	want = []string{"default-name (a, 1)", "default-name (a, 2)", "default-name (b, 1)", "default-name (b, 2)"}
			382	+	if got := names(`{"os": ["a", "b"], "n": [1, 2]}`); !reflect.DeepEqual(got, want) {
			383	+	t.Errorf("matrix-job-name: got %q, want %q", got, want)
			384	+	}
			385	+	// matrix-value-types: objects are not listed ("types 2.5 false" has a
			386	+	// custom name, so only the default name is checked here).
			387	+	if got := names(`{"n": [1], "obj": [{"k": "v"}]}`); !reflect.DeepEqual(got, []string{"default-name (1)"}) {
			388	+	t.Errorf("object value: got %q", got)
357		}	389		}
358	-	if got := combos[0].DisplayName("job"); got != "job (, 1)" {	390	+	}
359	-	t.Errorf("name = %q", got)	391	+
			392	+	// TestRecordedExcludeUnknownKey is matrix-edge-values a-exclude-unknown-key.yml:
			393	+	// exclude: [{nokey: x}, {os: one, nokey: y}] made the workflow invalid.
			394	+	func TestRecordedExcludeUnknownKey(t *testing.T) {
			395	+	_, err := ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"os": ["one", "two"], "exclude": [{"nokey": "x"}, {"os": "one", "nokey": "y"}]}}`))
			396	+	const one = "Matrix exclude key 'nokey' does not match any key within the matrix"
			397	+	if err == nil \|\| err.Error() != one+"\n"+one {
			398	+	t.Fatalf("err = %v, want the message once per offending entry", err)
			399	+	}
			400	+	// An entry that is fine does not add one.
			401	+	_, err = ExpandMatrix(ctxbg, jsonValue(t, `{"matrix": {"os": ["one", "two"], "exclude": [{"os": "one"}, {"nokey": "x"}]}}`))
			402	+	if err == nil \|\| err.Error() != one {
			403	+	t.Errorf("err = %v, want %q", err, one)
			404	+	}
			405	+	}
			406	+
			407	+	// TestRecordedMatrixOver256 is the scenario matrix-over-256: 17 x 16 = 272
			408	+	// configurations in job too-many made the workflow invalid.
			409	+	func TestRecordedMatrixOver256(t *testing.T) {
			410	+	w, errs := Parse("ci.yml", []byte(`on: push
			411	+	jobs:
			412	+	too-many:
			413	+	runs-on: x
			414	+	strategy:
			415	+	matrix:
			416	+	a: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]
			417	+	b: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
			418	+	steps: [{run: a}]
			419	+	`))
			420	+	if len(errs) > 0 {
			421	+	t.Fatal(errs)
			422	+	}
			423	+	_, err := w.Job("too-many").EvaluateStrategy(ctxbg, template.Env{})
			424	+	const want = "Strategy for job 'too-many' produced 272 configurations which exceeds the maximum of 256 configurations"
			425	+	if err == nil \|\| err.Error() != want {
			426	+	t.Errorf("err = %v, want %q", err, want)
			427	+	}
			428	+	var limit *ConfigurationLimitError
			429	+	if !errors.As(err, &limit) \|\| limit.Job != "too-many" \|\| limit.Count != 272 {
			430	+	t.Errorf("err = %#v", err)
360		}	431		}
361		}	432		}
362			433

M workflow/strategy.go +11 -5


@@ -134,7 +134,12 @@ func (j Job) EvaluateStrategy(ctx context.Context, env template.Env) (Strategy
134	134		if err != nil {
135	135		return nil, err
136	136		}
137		-	return ParseStrategy(ctx, v)
	137	+	st, err := ParseStrategy(ctx, v)
	138	+	var limit *ConfigurationLimitError
	139	+	if errors.As(err, &limit) {
	140	+	return nil, &ConfigurationLimitError{Job: j.ID, Count: limit.Count}
	141	+	}
	142	+	return st, err
138	143		}
139	144
140	145		// MatrixEnv returns env with the "matrix" and "strategy" contexts of

@@ -158,11 +163,12 @@ func (s *Strategy) MatrixEnv(env template.Env, index int) (template.Env, error)
158	163		// DisplayName evaluates the job's name for combination index of its
159	164		// strategy: the name token (or the job id) with the matrix values added in
160	165		// parentheses, unless the name is set and reads the matrix context, in which
161		-	// case it is used as is. env must already include the matrix and strategy
162		-	// contexts, see [Strategy.MatrixEnv].
	166	+	// case it is used as is. Null and empty values are not listed. env must
	167	+	// already include the matrix and strategy contexts, see [Strategy.MatrixEnv].
163	168		//
164		-	// TODO(verify with recording): GitHub adds the values to the name unless the
165		-	// name is set and refers to the matrix.
	169	+	// Recorded on github.com (matrix-job-name): "default-name (a, 1)", a custom
	170	+	// name using the matrix as is, and a custom name that does not read the matrix
	171	+	// followed by the values ("same name for every leg (1)").
166	172		func (j Job) DisplayName(ctx context.Context, env template.Env, s Strategy, index int) (string, error) {
167	173		if index < 0 \|\| index >= len(s.Combos) {
168	174		return "", fmt.Errorf("matrix combination %d out of range (%d combinations)", index, len(s.Combos))

 	if err != nil {
 		return nil, err
 	}
-	return ParseStrategy(ctx, v)
+	st, err := ParseStrategy(ctx, v)
+	var limit *ConfigurationLimitError
+	if errors.As(err, &limit) {
+		return nil, &ConfigurationLimitError{Job: j.ID, Count: limit.Count}
+	}
+	return st, err
 }
 // MatrixEnv returns env with the "matrix" and "strategy" contexts of
 // DisplayName evaluates the job's name for combination index of its
 // strategy: the name token (or the job id) with the matrix values added in
 // parentheses, unless the name is set and reads the matrix context, in which
-// case it is used as is. env must already include the matrix and strategy
+// case it is used as is. Null and empty values are not listed. env must
-// contexts, see [Strategy.MatrixEnv].
+// already include the matrix and strategy contexts, see [Strategy.MatrixEnv].
 //
-// TODO(verify with recording): GitHub adds the values to the name unless the
+// Recorded on github.com (matrix-job-name): "default-name (a, 1)", a custom
-// name is set and refers to the matrix.
+// name using the matrix as is, and a custom name that does not read the matrix
+// followed by the values ("same name for every leg (1)").
 func (j *Job) DisplayName(ctx context.Context, env template.Env, s *Strategy, index int) (string, error) {
 	if index < 0 || index >= len(s.Combos) {
 		return "", fmt.Errorf("matrix combination %d out of range (%d combinations)", index, len(s.Combos))