1// SPDX-License-Identifier: MIT
2
3package workflow
4
5import (
6 "encoding/json"
7 "fmt"
8 "io/fs"
9 "os"
10 "path"
11 "path/filepath"
12 "slices"
13 "strings"
14 "testing"
15
16 "go.yaml.in/yaml/v3"
17)
18
19// The tests in this file check SkipCI, MatchPush and Parse against the
20// scenarios and the recordings of ermite-conformance. They are opt-in: set
21// ERMITE_CORPUS to the directory that holds corpus/ and recordings/ (a
22// checkout of ermite-conformance). The files are only read.
23//
24// ERMITE_CORPUS=/path/to/ermite-conformance go test ./workflow -run Corpus -v
25
26type corpusPush struct {
27 Kind string `yaml:"kind"`
28 Tree string `yaml:"tree"`
29 Remove []string `yaml:"remove"`
30 Message string `yaml:"message"`
31 Count int `yaml:"count"`
32 Generate []struct {
33 Path string `yaml:"path"`
34 Count int `yaml:"count"`
35 } `yaml:"generate"`
36}
37
38type corpusScenario struct {
39 Pushes []corpusPush `yaml:"pushes"`
40}
41
42type corpusRecording struct {
43 Pushes []struct {
44 Index int `json:"index"`
45 Runs []struct {
46 Workflow string `json:"workflow"`
47 Conclusion string `json:"conclusion"`
48 Jobs []json.RawMessage `json:"jobs"`
49 } `json:"runs"`
50 } `json:"pushes"`
51}
52
53// readTree returns the files under dir by path relative to dir.
54func readTree(t *testing.T, dir string) map[string]string {
55 t.Helper()
56 files := map[string]string{}
57 err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
58 if err != nil || d.IsDir() {
59 return err
60 }
61 b, err := os.ReadFile(p)
62 if err != nil {
63 return err
64 }
65 rel, _ := filepath.Rel(dir, p)
66 files[filepath.ToSlash(rel)] = string(b)
67 return nil
68 })
69 if err != nil {
70 t.Fatal(err)
71 }
72 return files
73}
74
75func TestCorpusPushRules(t *testing.T) {
76 root := os.Getenv("ERMITE_CORPUS")
77 if root == "" {
78 t.Skip("ERMITE_CORPUS is not set")
79 }
80 dirs, err := filepath.Glob(filepath.Join(root, "corpus", "*", "scenario.yml"))
81 if err != nil || len(dirs) == 0 {
82 t.Fatalf("no scenario in %s/corpus: %v", root, err)
83 }
84 checked := 0
85 for _, sc := range dirs {
86 dir := filepath.Dir(sc)
87 name := filepath.Base(dir)
88 t.Run(name, func(t *testing.T) {
89 var s corpusScenario
90 b, err := os.ReadFile(sc)
91 if err != nil {
92 t.Fatal(err)
93 }
94 if err := yaml.Unmarshal(b, &s); err != nil {
95 t.Fatal(err)
96 }
97 for _, p := range s.Pushes {
98 if !slices.Contains([]string{"branch", "tag", "tags", "delete"}, p.Kind) {
99 t.Skipf("push kind %q is not modelled", p.Kind)
100 }
101 }
102 rb, err := os.ReadFile(filepath.Join(root, "recordings", name+".json"))
103 if err != nil {
104 t.Skipf("no recording: %v", err)
105 }
106 var rec corpusRecording
107 if err := json.Unmarshal(rb, &rec); err != nil {
108 t.Fatal(err)
109 }
110 checked++
111 checkScenario(t, dir, name, s, rec)
112 })
113 }
114 if checked == 0 {
115 t.Error("no scenario was checked")
116 }
117}
118
119func checkScenario(t *testing.T, dir, name string, s corpusScenario, rec corpusRecording) {
120 state := map[string]string{} // the files of the head commit
121 for i, p := range s.Pushes {
122 n := i + 1
123 var changed []string
124 if p.Kind != "delete" {
125 tree := p.Tree
126 if tree == "" && i == 0 {
127 tree = "tree"
128 }
129 next := cloneFiles(state)
130 if tree != "" {
131 for f, c := range readTree(t, filepath.Join(dir, tree)) {
132 next[f] = c
133 }
134 }
135 for _, f := range p.Remove {
136 delete(next, f)
137 }
138 for _, g := range p.Generate {
139 for k := 1; k <= g.Count; k++ {
140 next[strings.ReplaceAll(g.Path, "{n}", fmt.Sprint(k))] = "generated"
141 }
142 }
143 for f, c := range next {
144 if old, ok := state[f]; !ok || old != c {
145 changed = append(changed, f)
146 }
147 }
148 for f := range state {
149 if _, ok := next[f]; !ok {
150 changed = append(changed, f)
151 }
152 }
153 slices.Sort(changed)
154 state = next
155 }
156 message := p.Message
157 if message == "" {
158 message = fmt.Sprintf("scenario %s push %d", name, n)
159 }
160
161 info := PushInfo{Created: i == 0 && p.Kind == "branch", ChangedFiles: changed}
162 refs := 1
163 switch p.Kind {
164 case "branch":
165 info.Ref = "refs/heads/scn/" + name + "/r1"
166 case "tag", "tags":
167 refs = max(p.Count, 1)
168 info.Ref = "refs/tags/scn-" + name + "-r1-1"
169 info.TagCount = refs
170 info.Created = true
171 case "delete":
172 // Deletes the last ref pushed.
173 prev := s.Pushes[i-1]
174 info.Deleted = true
175 if prev.Kind == "branch" {
176 info.Ref = "refs/heads/scn/" + name + "/r1"
177 } else {
178 info.Ref = "refs/tags/scn-" + name + "-r1-1"
179 info.TagCount = 1
180 }
181 }
182
183 got := map[string]int{} // runs by workflow file in the recording
184 var failed []string // workflows recorded as a failed run without job
185 for _, rp := range rec.Pushes {
186 if rp.Index != n {
187 continue
188 }
189 for _, r := range rp.Runs {
190 got[r.Workflow]++
191 if r.Conclusion == "failure" && len(r.Jobs) == 0 {
192 failed = append(failed, r.Workflow)
193 }
194 }
195 }
196
197 skip := p.Kind != "delete" && SkipCI(message)
198 var files []string
199 for f := range state {
200 if strings.HasPrefix(f, ".github/workflows/") && path.Dir(f) == ".github/workflows" {
201 files = append(files, f)
202 }
203 }
204 slices.Sort(files)
205 for _, f := range files {
206 w, errs := Parse(f, []byte(state[f]))
207 want := 0
208 switch {
209 case p.Kind == "delete" || skip:
210 case len(errs) > 0:
211 // An invalid file gives one failed run without job.
212 want = 1
213 if !slices.Contains(failed, f) {
214 t.Errorf("push %d (%q): %s is invalid (%v) but the recording has no failed run for it", n, message, f, errs[0])
215 }
216 case w.MatchPush(info):
217 want = refs
218 }
219 if got[f] != want {
220 t.Errorf("push %d (%q, %s): %s: GitHub started %d run(s), we expect %d", n, message, p.Kind, f, got[f], want)
221 }
222 }
223 for f, c := range got {
224 if !slices.Contains(files, f) {
225 t.Errorf("push %d: recording has %d run(s) of %s, which is not in the scenario", n, c, f)
226 }
227 }
228 }
229}
230
231func cloneFiles(m map[string]string) map[string]string {
232 out := make(map[string]string, len(m))
233 for k, v := range m {
234 out[k] = v
235 }
236 return out
237}