detached b62d011

test(workflow): check SkipCI, MatchPush and Parse against the conformance corpus when ERMITE_CORPUS is set

commit
b62d01170fa2962dbd69956c92fa7f2e8ee93926
author
brice · · 2026-10-12 19:25 UTC
committer
brice · · 2026-10-18 14:59 UTC
parent
723958f
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
2 files · +242 -0
Split
UnifiedSplit
M workflow/PORTING.md +5 -0
@@ -42,6 +42,11 @@ code.
4242
4343 ## Recorded (github.com, ermite-conformance corpus)
4444
45+Opt-in check against the corpus itself: `ERMITE_CORPUS=<ermite-conformance
46+checkout> go test ./workflow -run Corpus` (`corpus_test.go`) replays the
47+scenario files and compares the runs SkipCI, MatchPush and Parse predict with
48+the recordings. The cases copied into the unit tests stay.
49+
4550 - Status functions (`success()`, `failure()`, `always()`, `cancelled()`) in
4651 `env` or `run` expressions make the workflow invalid (recorded:
4752 expr-status-in-env, expr-status-in-run, each a failed run with no job). The
@@ -42,6 +42,11 @@ code.
42 42
43 ## Recorded (github.com, ermite-conformance corpus)43 ## Recorded (github.com, ermite-conformance corpus)
44 44
45+Opt-in check against the corpus itself: `ERMITE_CORPUS=<ermite-conformance
46+checkout> go test ./workflow -run Corpus` (`corpus_test.go`) replays the
47+scenario files and compares the runs SkipCI, MatchPush and Parse predict with
48+the recordings. The cases copied into the unit tests stay.
49+
45 - Status functions (`success()`, `failure()`, `always()`, `cancelled()`) in50 - Status functions (`success()`, `failure()`, `always()`, `cancelled()`) in
46 `env` or `run` expressions make the workflow invalid (recorded:51 `env` or `run` expressions make the workflow invalid (recorded:
47 expr-status-in-env, expr-status-in-run, each a failed run with no job). The52 expr-status-in-env, expr-status-in-run, each a failed run with no job). The
A workflow/corpus_test.go +237 -0
@@ -0,0 +1,237 @@
1+// SPDX-License-Identifier: MIT
2+
3+package workflow
4+
5+import (
6+ "encoding/json"
7+ "fmt"
8+ "io/fs"
9+ "os"
10+ "path"
11+ "path/filepath"
12+ "slices"
13+ "strings"
14+ "testing"
15+
16+ "go.yaml.in/yaml/v3"
17+)
18+
19+// The tests in this file check SkipCI, MatchPush and Parse against the
20+// scenarios and the recordings of ermite-conformance. They are opt-in: set
21+// ERMITE_CORPUS to the directory that holds corpus/ and recordings/ (a
22+// checkout of ermite-conformance). The files are only read.
23+//
24+// ERMITE_CORPUS=/path/to/ermite-conformance go test ./workflow -run Corpus -v
25+
26+type corpusPush struct {
27+ Kind string `yaml:"kind"`
28+ Tree string `yaml:"tree"`
29+ Remove []string `yaml:"remove"`
30+ Message string `yaml:"message"`
31+ Count int `yaml:"count"`
32+ Generate []struct {
33+ Path string `yaml:"path"`
34+ Count int `yaml:"count"`
35+ } `yaml:"generate"`
36+}
37+
38+type corpusScenario struct {
39+ Pushes []corpusPush `yaml:"pushes"`
40+}
41+
42+type corpusRecording struct {
43+ Pushes []struct {
44+ Index int `json:"index"`
45+ Runs []struct {
46+ Workflow string `json:"workflow"`
47+ Conclusion string `json:"conclusion"`
48+ Jobs []json.RawMessage `json:"jobs"`
49+ } `json:"runs"`
50+ } `json:"pushes"`
51+}
52+
53+// readTree returns the files under dir by path relative to dir.
54+func readTree(t *testing.T, dir string) map[string]string {
55+ t.Helper()
56+ files := map[string]string{}
57+ err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
58+ if err != nil || d.IsDir() {
59+ return err
60+ }
61+ b, err := os.ReadFile(p)
62+ if err != nil {
63+ return err
64+ }
65+ rel, _ := filepath.Rel(dir, p)
66+ files[filepath.ToSlash(rel)] = string(b)
67+ return nil
68+ })
69+ if err != nil {
70+ t.Fatal(err)
71+ }
72+ return files
73+}
74+
75+func TestCorpusPushRules(t *testing.T) {
76+ root := os.Getenv("ERMITE_CORPUS")
77+ if root == "" {
78+ t.Skip("ERMITE_CORPUS is not set")
79+ }
80+ dirs, err := filepath.Glob(filepath.Join(root, "corpus", "*", "scenario.yml"))
81+ if err != nil || len(dirs) == 0 {
82+ t.Fatalf("no scenario in %s/corpus: %v", root, err)
83+ }
84+ checked := 0
85+ for _, sc := range dirs {
86+ dir := filepath.Dir(sc)
87+ name := filepath.Base(dir)
88+ t.Run(name, func(t *testing.T) {
89+ var s corpusScenario
90+ b, err := os.ReadFile(sc)
91+ if err != nil {
92+ t.Fatal(err)
93+ }
94+ if err := yaml.Unmarshal(b, &s); err != nil {
95+ t.Fatal(err)
96+ }
97+ for _, p := range s.Pushes {
98+ if !slices.Contains([]string{"branch", "tag", "tags", "delete"}, p.Kind) {
99+ t.Skipf("push kind %q is not modelled", p.Kind)
100+ }
101+ }
102+ rb, err := os.ReadFile(filepath.Join(root, "recordings", name+".json"))
103+ if err != nil {
104+ t.Skipf("no recording: %v", err)
105+ }
106+ var rec corpusRecording
107+ if err := json.Unmarshal(rb, &rec); err != nil {
108+ t.Fatal(err)
109+ }
110+ checked++
111+ checkScenario(t, dir, name, s, rec)
112+ })
113+ }
114+ if checked == 0 {
115+ t.Error("no scenario was checked")
116+ }
117+}
118+
119+func checkScenario(t *testing.T, dir, name string, s corpusScenario, rec corpusRecording) {
120+ state := map[string]string{} // the files of the head commit
121+ for i, p := range s.Pushes {
122+ n := i + 1
123+ var changed []string
124+ if p.Kind != "delete" {
125+ tree := p.Tree
126+ if tree == "" && i == 0 {
127+ tree = "tree"
128+ }
129+ next := cloneFiles(state)
130+ if tree != "" {
131+ for f, c := range readTree(t, filepath.Join(dir, tree)) {
132+ next[f] = c
133+ }
134+ }
135+ for _, f := range p.Remove {
136+ delete(next, f)
137+ }
138+ for _, g := range p.Generate {
139+ for k := 1; k <= g.Count; k++ {
140+ next[strings.ReplaceAll(g.Path, "{n}", fmt.Sprint(k))] = "generated"
141+ }
142+ }
143+ for f, c := range next {
144+ if old, ok := state[f]; !ok || old != c {
145+ changed = append(changed, f)
146+ }
147+ }
148+ for f := range state {
149+ if _, ok := next[f]; !ok {
150+ changed = append(changed, f)
151+ }
152+ }
153+ slices.Sort(changed)
154+ state = next
155+ }
156+ message := p.Message
157+ if message == "" {
158+ message = fmt.Sprintf("scenario %s push %d", name, n)
159+ }
160+
161+ info := PushInfo{Created: i == 0 && p.Kind == "branch", ChangedFiles: changed}
162+ refs := 1
163+ switch p.Kind {
164+ case "branch":
165+ info.Ref = "refs/heads/scn/" + name + "/r1"
166+ case "tag", "tags":
167+ refs = max(p.Count, 1)
168+ info.Ref = "refs/tags/scn-" + name + "-r1-1"
169+ info.TagCount = refs
170+ info.Created = true
171+ case "delete":
172+ // Deletes the last ref pushed.
173+ prev := s.Pushes[i-1]
174+ info.Deleted = true
175+ if prev.Kind == "branch" {
176+ info.Ref = "refs/heads/scn/" + name + "/r1"
177+ } else {
178+ info.Ref = "refs/tags/scn-" + name + "-r1-1"
179+ info.TagCount = 1
180+ }
181+ }
182+
183+ got := map[string]int{} // runs by workflow file in the recording
184+ var failed []string // workflows recorded as a failed run without job
185+ for _, rp := range rec.Pushes {
186+ if rp.Index != n {
187+ continue
188+ }
189+ for _, r := range rp.Runs {
190+ got[r.Workflow]++
191+ if r.Conclusion == "failure" && len(r.Jobs) == 0 {
192+ failed = append(failed, r.Workflow)
193+ }
194+ }
195+ }
196+
197+ skip := p.Kind != "delete" && SkipCI(message)
198+ var files []string
199+ for f := range state {
200+ if strings.HasPrefix(f, ".github/workflows/") && path.Dir(f) == ".github/workflows" {
201+ files = append(files, f)
202+ }
203+ }
204+ slices.Sort(files)
205+ for _, f := range files {
206+ w, errs := Parse(f, []byte(state[f]))
207+ want := 0
208+ switch {
209+ case p.Kind == "delete" || skip:
210+ case len(errs) > 0:
211+ // An invalid file gives one failed run without job.
212+ want = 1
213+ if !slices.Contains(failed, f) {
214+ t.Errorf("push %d (%q): %s is invalid (%v) but the recording has no failed run for it", n, message, f, errs[0])
215+ }
216+ case w.MatchPush(info):
217+ want = refs
218+ }
219+ if got[f] != want {
220+ t.Errorf("push %d (%q, %s): %s: GitHub started %d run(s), we expect %d", n, message, p.Kind, f, got[f], want)
221+ }
222+ }
223+ for f, c := range got {
224+ if !slices.Contains(files, f) {
225+ t.Errorf("push %d: recording has %d run(s) of %s, which is not in the scenario", n, c, f)
226+ }
227+ }
228+ }
229+}
230+
231+func cloneFiles(m map[string]string) map[string]string {
232+ out := make(map[string]string, len(m))
233+ for k, v := range m {
234+ out[k] = v
235+ }
236+ return out
237+}
@@ -0,0 +1,237 @@
1+// SPDX-License-Identifier: MIT
2+
3+package workflow
4+
5+import (
6+ "encoding/json"
7+ "fmt"
8+ "io/fs"
9+ "os"
10+ "path"
11+ "path/filepath"
12+ "slices"
13+ "strings"
14+ "testing"
15+
16+ "go.yaml.in/yaml/v3"
17+)
18+
19+// The tests in this file check SkipCI, MatchPush and Parse against the
20+// scenarios and the recordings of ermite-conformance. They are opt-in: set
21+// ERMITE_CORPUS to the directory that holds corpus/ and recordings/ (a
22+// checkout of ermite-conformance). The files are only read.
23+//
24+// ERMITE_CORPUS=/path/to/ermite-conformance go test ./workflow -run Corpus -v
25+
26+type corpusPush struct {
27+ Kind string `yaml:"kind"`
28+ Tree string `yaml:"tree"`
29+ Remove []string `yaml:"remove"`
30+ Message string `yaml:"message"`
31+ Count int `yaml:"count"`
32+ Generate []struct {
33+ Path string `yaml:"path"`
34+ Count int `yaml:"count"`
35+ } `yaml:"generate"`
36+}
37+
38+type corpusScenario struct {
39+ Pushes []corpusPush `yaml:"pushes"`
40+}
41+
42+type corpusRecording struct {
43+ Pushes []struct {
44+ Index int `json:"index"`
45+ Runs []struct {
46+ Workflow string `json:"workflow"`
47+ Conclusion string `json:"conclusion"`
48+ Jobs []json.RawMessage `json:"jobs"`
49+ } `json:"runs"`
50+ } `json:"pushes"`
51+}
52+
53+// readTree returns the files under dir by path relative to dir.
54+func readTree(t *testing.T, dir string) map[string]string {
55+ t.Helper()
56+ files := map[string]string{}
57+ err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
58+ if err != nil || d.IsDir() {
59+ return err
60+ }
61+ b, err := os.ReadFile(p)
62+ if err != nil {
63+ return err
64+ }
65+ rel, _ := filepath.Rel(dir, p)
66+ files[filepath.ToSlash(rel)] = string(b)
67+ return nil
68+ })
69+ if err != nil {
70+ t.Fatal(err)
71+ }
72+ return files
73+}
74+
75+func TestCorpusPushRules(t *testing.T) {
76+ root := os.Getenv("ERMITE_CORPUS")
77+ if root == "" {
78+ t.Skip("ERMITE_CORPUS is not set")
79+ }
80+ dirs, err := filepath.Glob(filepath.Join(root, "corpus", "*", "scenario.yml"))
81+ if err != nil || len(dirs) == 0 {
82+ t.Fatalf("no scenario in %s/corpus: %v", root, err)
83+ }
84+ checked := 0
85+ for _, sc := range dirs {
86+ dir := filepath.Dir(sc)
87+ name := filepath.Base(dir)
88+ t.Run(name, func(t *testing.T) {
89+ var s corpusScenario
90+ b, err := os.ReadFile(sc)
91+ if err != nil {
92+ t.Fatal(err)
93+ }
94+ if err := yaml.Unmarshal(b, &s); err != nil {
95+ t.Fatal(err)
96+ }
97+ for _, p := range s.Pushes {
98+ if !slices.Contains([]string{"branch", "tag", "tags", "delete"}, p.Kind) {
99+ t.Skipf("push kind %q is not modelled", p.Kind)
100+ }
101+ }
102+ rb, err := os.ReadFile(filepath.Join(root, "recordings", name+".json"))
103+ if err != nil {
104+ t.Skipf("no recording: %v", err)
105+ }
106+ var rec corpusRecording
107+ if err := json.Unmarshal(rb, &rec); err != nil {
108+ t.Fatal(err)
109+ }
110+ checked++
111+ checkScenario(t, dir, name, s, rec)
112+ })
113+ }
114+ if checked == 0 {
115+ t.Error("no scenario was checked")
116+ }
117+}
118+
119+func checkScenario(t *testing.T, dir, name string, s corpusScenario, rec corpusRecording) {
120+ state := map[string]string{} // the files of the head commit
121+ for i, p := range s.Pushes {
122+ n := i + 1
123+ var changed []string
124+ if p.Kind != "delete" {
125+ tree := p.Tree
126+ if tree == "" && i == 0 {
127+ tree = "tree"
128+ }
129+ next := cloneFiles(state)
130+ if tree != "" {
131+ for f, c := range readTree(t, filepath.Join(dir, tree)) {
132+ next[f] = c
133+ }
134+ }
135+ for _, f := range p.Remove {
136+ delete(next, f)
137+ }
138+ for _, g := range p.Generate {
139+ for k := 1; k <= g.Count; k++ {
140+ next[strings.ReplaceAll(g.Path, "{n}", fmt.Sprint(k))] = "generated"
141+ }
142+ }
143+ for f, c := range next {
144+ if old, ok := state[f]; !ok || old != c {
145+ changed = append(changed, f)
146+ }
147+ }
148+ for f := range state {
149+ if _, ok := next[f]; !ok {
150+ changed = append(changed, f)
151+ }
152+ }
153+ slices.Sort(changed)
154+ state = next
155+ }
156+ message := p.Message
157+ if message == "" {
158+ message = fmt.Sprintf("scenario %s push %d", name, n)
159+ }
160+
161+ info := PushInfo{Created: i == 0 && p.Kind == "branch", ChangedFiles: changed}
162+ refs := 1
163+ switch p.Kind {
164+ case "branch":
165+ info.Ref = "refs/heads/scn/" + name + "/r1"
166+ case "tag", "tags":
167+ refs = max(p.Count, 1)
168+ info.Ref = "refs/tags/scn-" + name + "-r1-1"
169+ info.TagCount = refs
170+ info.Created = true
171+ case "delete":
172+ // Deletes the last ref pushed.
173+ prev := s.Pushes[i-1]
174+ info.Deleted = true
175+ if prev.Kind == "branch" {
176+ info.Ref = "refs/heads/scn/" + name + "/r1"
177+ } else {
178+ info.Ref = "refs/tags/scn-" + name + "-r1-1"
179+ info.TagCount = 1
180+ }
181+ }
182+
183+ got := map[string]int{} // runs by workflow file in the recording
184+ var failed []string // workflows recorded as a failed run without job
185+ for _, rp := range rec.Pushes {
186+ if rp.Index != n {
187+ continue
188+ }
189+ for _, r := range rp.Runs {
190+ got[r.Workflow]++
191+ if r.Conclusion == "failure" && len(r.Jobs) == 0 {
192+ failed = append(failed, r.Workflow)
193+ }
194+ }
195+ }
196+
197+ skip := p.Kind != "delete" && SkipCI(message)
198+ var files []string
199+ for f := range state {
200+ if strings.HasPrefix(f, ".github/workflows/") && path.Dir(f) == ".github/workflows" {
201+ files = append(files, f)
202+ }
203+ }
204+ slices.Sort(files)
205+ for _, f := range files {
206+ w, errs := Parse(f, []byte(state[f]))
207+ want := 0
208+ switch {
209+ case p.Kind == "delete" || skip:
210+ case len(errs) > 0:
211+ // An invalid file gives one failed run without job.
212+ want = 1
213+ if !slices.Contains(failed, f) {
214+ t.Errorf("push %d (%q): %s is invalid (%v) but the recording has no failed run for it", n, message, f, errs[0])
215+ }
216+ case w.MatchPush(info):
217+ want = refs
218+ }
219+ if got[f] != want {
220+ t.Errorf("push %d (%q, %s): %s: GitHub started %d run(s), we expect %d", n, message, p.Kind, f, got[f], want)
221+ }
222+ }
223+ for f, c := range got {
224+ if !slices.Contains(files, f) {
225+ t.Errorf("push %d: recording has %d run(s) of %s, which is not in the scenario", n, c, f)
226+ }
227+ }
228+ }
229+}
230+
231+func cloneFiles(m map[string]string) map[string]string {
232+ out := make(map[string]string, len(m))
233+ for k, v := range m {
234+ out[k] = v
235+ }
236+ return out
237+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 1

Tags 3

↑↓ moveenter openesc close