1// SPDX-License-Identifier: MIT
2
3package executor
4
5import (
6 "fmt"
7 "strings"
8 "sync"
9 "testing"
10)
11
12// resultRecorder is a recorder that also keeps StepResults.
13type resultRecorder struct {
14 *recorder
15 rmu sync.Mutex
16 results map[int]StepResults
17}
18
19func newResultRecorder() *resultRecorder {
20 return &resultRecorder{recorder: newRecorder(), results: map[int]StepResults{}}
21}
22
23func (r *resultRecorder) StepResults(n int, res StepResults) {
24 r.rmu.Lock()
25 defer r.rmu.Unlock()
26 r.results[n] = res
27}
28
29func runResults(t *testing.T, yaml string, tweak func(*Config)) (*resultRecorder, Result) {
30 t.Helper()
31 rr := newResultRecorder()
32 // The recorder is wrapped: Reporter methods come from recorder, StepResults
33 // from resultRecorder.
34 _, res := runLocal(t, yaml, "j", func(c *Config) {
35 c.Reporter = rr
36 c.SendAnnotations, c.SendSummaries = true, true
37 if tweak != nil {
38 tweak(c)
39 }
40 })
41 return rr, res
42}
43
44// Cases of ActionCommandManagerL0.IssueCommandInvalidColumns (actions/runner,
45// MIT), as a table: the properties after ValidateLinesAndColumns.
46func TestValidateLinesAndColumns(t *testing.T) {
47 reg := func(string) bool { return true }
48 for _, tc := range []struct {
49 name, cmd string
50 want map[string]string
51 }{
52 {"columns when lines differ", "::warning line=1,endLine=2,col=1,endColumn=2::m", map[string]string{"line": "1", "endline": "2"}},
53 {"no lines with columns", "::warning col=1,endColumn=2::m", map[string]string{}},
54 {"no line with endLine", "::warning endLine=1::m", map[string]string{"line": "1", "endline": "1"}},
55 {"no column with endColumn", "::warning line=1,endColumn=2::m", map[string]string{"line": "1", "col": "2", "endcolumn": "2"}},
56 {"empty strings", "::warning line=,endLine=3::m", map[string]string{"line": "3", "endline": "3"}},
57 {"end line before line", "::warning line=4,endLine=3::m", map[string]string{}},
58 {"end column before column", "::warning line=1,endLine=1,col=3,endColumn=2::m", map[string]string{"line": "1", "endline": "1"}},
59 {"valid", "::warning line=1,endLine=1,col=1,endColumn=2::m", map[string]string{"line": "1", "endline": "1", "col": "1", "endcolumn": "2"}},
60 {"backwards compatibility", "::warning line=1,col=1,file=test.txt::m", map[string]string{"line": "1", "col": "1", "file": "test.txt"}},
61 } {
62 c, ok := parseCommandV2(tc.cmd, reg)
63 if !ok {
64 t.Fatalf("%s: not parsed", tc.name)
65 }
66 props := map[string]string{}
67 for k, v := range c.Props {
68 props[k] = v
69 }
70 validateLinesAndColumns(props, "warning", func(string) {})
71 if fmt.Sprint(props) != fmt.Sprint(tc.want) {
72 t.Errorf("%s: %v, want %v", tc.name, props, tc.want)
73 }
74 }
75}
76
77func TestAnnotationsFromCommands(t *testing.T) {
78 needTools(t, "bash")
79 rr, res := runResults(t, wfHead+` steps:
80 - run: |
81 echo "::warning title=T,file=src/a.go,line=3,endLine=5,col=1,endColumn=2::careful"
82 echo "::error::broken%0Aline two"
83 echo "::notice::fyi"
84`, nil)
85 if res.Conclusion != ConclusionSuccess {
86 t.Fatalf("%+v", res)
87 }
88 got := rr.results[2].Annotations
89 want := []Annotation{
90 {Level: LevelWarning, Title: "T", Message: "careful", File: "src/a.go", StartLine: 3, EndLine: 5},
91 // No file and no line: the line of the ##[error] and ##[notice] lines in
92 // the step log (header: group, 3 script lines, shell, endgroup = 6 lines;
93 // the warning is line 7, the error 8 and its second line 9, the notice 10).
94 {Level: LevelFailure, Message: "broken\nline two", StartLine: 8, EndLine: 8},
95 {Level: LevelNotice, Message: "fyi", StartLine: 10, EndLine: 10},
96 }
97 // col and endColumn are dropped because the lines differ.
98 if fmt.Sprint(got) != fmt.Sprint(want) {
99 t.Errorf("annotations\n got %+v\nwant %+v", got, want)
100 }
101 log := logOf(rr.step(t, "Run echo \"::warning title=T,file=src/a.go,line=3,endLine=5,col=1,endColumn=2::careful\""))
102 for _, w := range []string{"##[warning]careful", "##[error]broken\nline two", "##[notice]fyi"} {
103 if !strings.Contains(log, w) {
104 t.Errorf("log lacks %q:\n%s", w, log)
105 }
106 }
107}
108
109// ExecutionContext.cs:144-146: ten per level per step, messages of 4096
110// characters at most; the rest only reaches the log.
111func TestAnnotationLimits(t *testing.T) {
112 needTools(t, "bash")
113 rr, _ := runResults(t, wfHead+` steps:
114 - name: many
115 run: |
116 for i in $(seq 1 12); do
117 echo "::warning::w$i"; echo "::error::e$i"; echo "::notice::n$i"
118 done
119 - name: long
120 run: |
121 printf '::warning::%s\n' "$(head -c 5000 /dev/zero | tr '\0' x)"
122`, nil)
123 count := map[AnnotationLevel]int{}
124 for _, a := range rr.results[2].Annotations {
125 count[a.Level]++
126 }
127 if count[LevelWarning] != 10 || count[LevelFailure] != 10 || count[LevelNotice] != 10 {
128 t.Errorf("counts %v", count)
129 }
130 if a := rr.results[2].Annotations; a[len(a)-1].Message == "w12" {
131 t.Error("the eleventh was kept")
132 }
133 log := logOf(rr.step(t, "many"))
134 if !strings.Contains(log, "##[warning]w12") || !strings.Contains(log, "##[error]e12") {
135 t.Error("annotations over the limit are still logged")
136 }
137 long := rr.results[3].Annotations
138 if len(long) != 1 || len(long[0].Message) != 4096 {
139 t.Errorf("long message: %d annotations, len %d", len(long), len(long[0].Message))
140 }
141 if l := logOf(rr.step(t, "long")); !strings.Contains(l, "##[warning]"+strings.Repeat("x", 4096)+"\n") && !strings.HasSuffix(l, "##[warning]"+strings.Repeat("x", 4096)) {
142 t.Error("the log has the message cut to 4096 characters")
143 }
144}
145
146// Every ##[error] the runner itself writes is an issue and an annotation.
147func TestRunnerErrorsAreAnnotations(t *testing.T) {
148 needTools(t, "bash")
149 rr, res := runResults(t, wfHead+` steps:
150 - run: exit 3
151`, nil)
152 if res.Conclusion != ConclusionFailure {
153 t.Fatal(res)
154 }
155 a := rr.results[2].Annotations
156 if len(a) != 1 || a[0].Level != LevelFailure || a[0].Message != "Process completed with exit code 3." {
157 t.Errorf("%+v", a)
158 }
159}
160
161func TestAnnotationsAreMaskedAndOffWithoutTheFeature(t *testing.T) {
162 needTools(t, "bash")
163 rr, _ := runResults(t, wfHead+` steps:
164 - run: |
165 echo "::add-mask::hunter2hunter2"
166 echo "::warning title=hunter2hunter2::pw hunter2hunter2"
167`, nil)
168 a := rr.results[2].Annotations
169 if len(a) != 1 || a[0].Message != "pw ***" || a[0].Title != "***" {
170 t.Errorf("%+v", a)
171 }
172 rr2, _ := runResults(t, wfHead+" steps:\n - run: echo '::warning::w'\n", func(c *Config) { c.SendAnnotations = false })
173 if len(rr2.results[2].Annotations) != 0 {
174 t.Error("annotations without the feature")
175 }
176}
177
178func TestAddMaskCommand(t *testing.T) {
179 needTools(t, "bash")
180 rec, res := runLocal(t, wfHead+` steps:
181 - name: step one
182 run: |
183 echo "::add-mask::abc%0Ddef%0Aghi%0D%0Ajkl"
184 echo "::add-mask:: %0D %0A %0D%0A %0D"
185 echo "::add-mask::"
186 echo "def ghi jkl abc"
187 echo "::echo::on"
188 echo "::add-mask::echoed-value"
189 echo "echoed-value"
190 - name: step two
191 run: echo "still masked jkl"
192`, "j", nil)
193 if res.Conclusion != ConclusionSuccess {
194 t.Fatalf("%+v", res)
195 }
196 log := logOf(rec.step(t, "step one"))
197 _, body, _ := strings.Cut(log, "##[endgroup]\n")
198 for _, w := range []string{"*** *** *** ***", "##[warning]Can't add secret mask for empty string in ##[add-mask] command.", "::add-mask::***", "\n***\n"} {
199 if !strings.Contains(body+"\n", w) {
200 t.Errorf("log lacks %q:\n%s", w, body)
201 }
202 }
203 if strings.Contains(body, "echoed-value") && !strings.Contains(body, "::echo::on") {
204 t.Errorf("%s", body)
205 }
206 if got := logOf(rec.step(t, "step two")); !strings.HasSuffix(got, "still masked ***") {
207 t.Errorf("%s", got)
208 }
209}
210
211func TestSaveStateInRunStep(t *testing.T) {
212 needTools(t, "bash")
213 var j *job
214 rec, res := runLocal(t, wfHead+` steps:
215 - run: |
216 echo "::save-state name=Foo::bar"
217 echo "baz=qux" >> "$GITHUB_STATE"
218 echo "::save-state::x"
219`, "j", func(c *Config) { _ = j })
220 if res.Conclusion != ConclusionFailure {
221 t.Fatalf("a save-state without a name fails the step: %+v", res)
222 }
223 log := logOf(rec.steps[1])
224 for _, w := range []string{"##[warning]The `save-state` command is deprecated", "##[error]Required field 'name' is missing in ##[save-state] command."} {
225 if !strings.Contains(log, w) {
226 t.Errorf("log lacks %q:\n%s", w, log)
227 }
228 }
229}
230
231// IssueExtensions.ToAnnotation (Sdk/RSWebApi/Contracts/IssueExtensions.cs:25-29):
232// an issue without file and line takes the number of its own line in the step
233// log (logFileLineNumber, ExecutionContext.cs:855) as start and end line.
234func TestAnnotationOfAPathlessIssueIsItsLogLine(t *testing.T) {
235 j := &job{}
236 j.masker.Store(newMasker())
237 for _, tc := range []struct {
238 name string
239 props map[string]string
240 logLine int
241 want [2]int32
242 }{
243 {"no props", nil, 103, [2]int32{103, 103}},
244 {"not logged", nil, 0, [2]int32{0, 0}},
245 {"blank file", map[string]string{"file": " "}, 5, [2]int32{5, 5}},
246 {"zero line", map[string]string{"line": "0"}, 5, [2]int32{5, 5}},
247 {"line not a number", map[string]string{"line": "x"}, 5, [2]int32{5, 5}},
248 {"a file", map[string]string{"file": "a.go"}, 5, [2]int32{0, 0}},
249 {"a line", map[string]string{"line": "2"}, 5, [2]int32{2, 2}},
250 {"a line range", map[string]string{"line": "2", "endline": "4"}, 5, [2]int32{2, 4}},
251 {"negative line, no file", map[string]string{"line": "-1"}, 5, [2]int32{0, 0}},
252 } {
253 a := j.annotationOf(issue{Level: LevelWarning, Props: tc.props}, "m", tc.logLine)
254 wantFile := tc.props["file"]
255 if strings.TrimSpace(wantFile) == "" {
256 wantFile = ""
257 }
258 if got := [2]int32{a.StartLine, a.EndLine}; got != tc.want || a.File != wantFile {
259 t.Errorf("%s: %+v, want lines %v", tc.name, a, tc.want)
260 }
261 }
262}
263
264// ann-*: a plain ::warning:: gets the number of its line in the log of the step,
265// the header counted (recorded: 103 for the exit code error closing a 103 line
266// log). Here: group, script (4 lines), shell, endgroup, then the output.
267func TestPlainWarningLineIsItsLogLine(t *testing.T) {
268 needTools(t, "bash")
269 rr, _ := runResults(t, wfHead+` steps:
270 - name: s
271 run: |
272 echo out
273 echo "::warning::x"
274 echo "::warning::x"
275 exit 3
276`, nil)
277 log := rr.step(t, "s").Log
278 var want []int32
279 for i, l := range log {
280 if strings.HasPrefix(l, "##[warning]") || strings.HasPrefix(l, "##[error]") {
281 want = append(want, int32(i+1))
282 }
283 }
284 if fmt.Sprint(want) != "[9 10 11]" {
285 t.Fatalf("log layout changed: %v\n%s", want, strings.Join(log, "\n"))
286 }
287 var got []int32
288 for _, a := range rr.results[2].Annotations {
289 if a.File != "" || a.StartLine != a.EndLine {
290 t.Errorf("%+v", a)
291 }
292 got = append(got, a.StartLine)
293 }
294 if fmt.Sprint(got) != fmt.Sprint(want) {
295 t.Errorf("start lines %v, log lines %v\n%s", got, want, strings.Join(log, "\n"))
296 }
297}
298
299// Mirror of the recording match-catastrophic (GitHub): the annotations of a
300// matcher with no file are at the line of their ##[warning] line, 10, 13, 16,
301// 18 and 20 in the steps of the corpus script (the removal warning at 14 and,
302// in the next step, 7). The step logs are laid out as the recorded ones.
303func TestMatcherAnnotationsWithoutFileHaveTheirLogLine(t *testing.T) {
304 needTools(t, "bash")
305 rr, res := runResults(t, wfHead+` steps:
306 - name: workspace as a repository
307 run: |
308 mkdir -p src .github
309 cat > .github/bad.json <<'JSON'
310 {"problemMatcher":[{"owner":"catastrophic","pattern":[{"regexp":"^((a+)+)$","message":1}]}]}
311 JSON
312 cat > .github/good.json <<'JSON'
313 {"problemMatcher":[{"owner":"good","severity":"warning","pattern":[{"regexp":"^good: (.+)$","message":1}]}]}
314 JSON
315 printf 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!\n' > .github/long-line.txt
316 - name: catastrophic matcher
317 run: |
318 echo "::add-matcher::.github/bad.json"
319 echo "::add-matcher::.github/good.json"
320 echo "good: before the long line"
321 cat .github/long-line.txt
322 echo "good: after the long line"
323 echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!"
324 echo "good: after a second long line"
325 echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
326 echo "good: matcher still alive"
327 - name: the matcher stays removed in the next step
328 run: |
329 echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!"
330 echo "aaa"
331 echo "good: still alive"
332`, nil)
333 if res.Conclusion != ConclusionSuccess {
334 t.Fatalf("%+v", res)
335 }
336 lines := func(num int) (out []string) {
337 for _, a := range rr.results[num].Annotations {
338 if a.File != "" || a.StartLine != a.EndLine {
339 t.Errorf("step %d: %+v", num, a)
340 }
341 m, _, _ := strings.Cut(a.Message, ".")
342 out = append(out, fmt.Sprintf("%d:%s", a.StartLine, m))
343 }
344 return out
345 }
346 // The first step has no annotation; the removal message is line 14.
347 want3 := "[13:before the long line 14:Removing issue matcher 'catastrophic'" +
348 " 16:after the long line 18:after a second long line 20:matcher still alive]"
349 if got := fmt.Sprint(lines(3)); got != want3 {
350 t.Errorf("catastrophic matcher: %v\nwant %v\n%s", got, want3, logOf(rr.step(t, "catastrophic matcher")))
351 }
352 want4 := "[7:Removing issue matcher 'catastrophic' 10:still alive]"
353 if got := fmt.Sprint(lines(4)); got != want4 {
354 t.Errorf("next step: %v\nwant %v\n%s", got, want4, logOf(rr.step(t, "the matcher stays removed in the next step")))
355 }
356}
357
358// A title is bounded like the message, at 4096 characters (GitHub has no cap;
359// ann-limits records a 600 character title): a 600 one is sent intact.
360func TestAnnotationTitleIsSentIntactUpTo4096(t *testing.T) {
361 needTools(t, "bash")
362 rr, _ := runResults(t, wfHead+` steps:
363 - run: |
364 t600=$(printf 't%.0s' $(seq 600))
365 t4100=$(printf 'u%.0s' $(seq 4100))
366 echo "::warning title=$t600::m600"
367 echo "::warning title=$t4100::m4100"
368`, nil)
369 got := map[string]string{}
370 for _, a := range rr.results[2].Annotations {
371 got[a.Message] = a.Title
372 }
373 if got["m600"] != strings.Repeat("t", 600) {
374 t.Errorf("600 character title: %d characters", len([]rune(got["m600"])))
375 }
376 if got["m4100"] != strings.Repeat("u", 4096) {
377 t.Errorf("4100 character title: %d characters", len([]rune(got["m4100"])))
378 }
379}