1// SPDX-License-Identifier: MIT
2
3package executor
4
5import (
6 "os"
7 "path/filepath"
8 "strings"
9 "testing"
10 "time"
11)
12
13func fastTimeouts(c *Config) {
14 c.Actions.Containers = ContainerTimeouts{Command: 400 * time.Millisecond, Pull: 400 * time.Millisecond, CleanupCommand: 400 * time.Millisecond, CleanupBudget: 3 * time.Second}
15}
16
17func names(l [][]string) string {
18 var v []string
19 for _, c := range l {
20 v = append(v, c[0])
21 }
22 return strings.Join(v, " ")
23}
24
25// GO-001: a docker command that stalls is stopped by its own limit, the job
26// does not wait for its timeout, and the cleanup still finds the container
27// that the daemon did create, by the label of the job.
28func TestStalledCreateIsBoundedAndCleanedByLabel(t *testing.T) {
29 f := newFakeCtr(t)
30 f.set(t, "hang_create_after", "1") // created in the daemon, the CLI never returns
31 start := time.Now()
32 rec, res := ctrRun(t, f, ctrJob, " - run: echo never\n", fastTimeouts)
33 if res.Conclusion != ConclusionFailure || time.Since(start) > 20*time.Second {
34 t.Fatalf("%+v after %s", res, time.Since(start))
35 }
36 log := logOf(rec.step(t, "Initialize containers"))
37 if !strings.Contains(log, "##[error]The docker command timed out after 400ms: create --name ") {
38 t.Errorf("init:\n%s", log)
39 }
40 if b, _ := os.ReadFile(filepath.Join(f.dir, "created")); strings.TrimSpace(string(b)) != "" {
41 t.Errorf("container left in the daemon: %q", b)
42 }
43 if got := logOf(rec.step(t, "Stop containers")); !strings.Contains(got, "Remove container network: github_network_") {
44 t.Errorf("stop:\n%s", got)
45 }
46}
47
48// GO-003-R: options: may carry another --name (the runner passes it on). The
49// container is then not named as recorded; it is found by the label.
50func TestOverridingNameAndTimedOutCreateAreCleanedByLabel(t *testing.T) {
51 f := newFakeCtr(t)
52 f.set(t, "hang_create_after", "1")
53 rec, _ := ctrRun(t, f, " container:\n image: ubuntu:24.04\n options: --name my-own-name\n", " - run: echo never\n", fastTimeouts)
54 var ps, rm []string
55 for _, c := range f.calls(t) {
56 if c[0] == "ps" && strings.Contains(strings.Join(c, " "), "--quiet") {
57 ps = c
58 }
59 if c[0] == "rm" {
60 rm = c
61 }
62 }
63 if ps == nil || !strings.Contains(strings.Join(ps, "\n"), "label=") {
64 t.Errorf("no listing by label: %v", ps)
65 }
66 if len(rm) < 3 || !containerIDRe.MatchString(rm[2]) {
67 t.Errorf("rm by id expected: %v", rm)
68 }
69 if b, _ := os.ReadFile(filepath.Join(f.dir, "created")); strings.TrimSpace(string(b)) != "" {
70 t.Errorf("container left: %q", b)
71 }
72 _ = rec
73}
74
75// GO-001: the cancellation of the job reaches every docker call of the init.
76func TestCancelDuringInitStopsTheCommandAndCleans(t *testing.T) {
77 for _, verb := range []string{"version", "inspect", "port", "start", "ps"} {
78 f := newFakeCtr(t)
79 f.set(t, "hang_"+verb, "1")
80 cancel := NewCanceller()
81 go func() { time.Sleep(500 * time.Millisecond); cancel.Cancel(CancelUser) }()
82 start := time.Now()
83 rec, res := ctrRunC(t, f, cancel, " container: ubuntu:24.04\n services:\n s:\n image: redis:7\n", " - run: echo never\n", func(c *Config) {
84 c.Actions.Containers.Command = time.Minute
85 })
86 if res.Conclusion != ConclusionCancelled || time.Since(start) > 15*time.Second {
87 t.Fatalf("%s: %+v after %s %v", verb, res, time.Since(start), rec.names())
88 }
89 if got := rec.concl("Stop containers"); got != ConclusionSuccess {
90 t.Errorf("%s: Stop containers %s", verb, got)
91 }
92 }
93}
94
95// GO-002: one removal that stalls does not keep the others and the network from
96// being removed; the logs of a service never prevent its removal; the
97// failures are reported by the step.
98func TestCleanupAttemptsEverythingWithItsOwnBudget(t *testing.T) {
99 f := newFakeCtr(t)
100 f.set(t, "hang_logs", "1")
101 f.set(t, "hang_for", "rm --force "+strings.Repeat("0", 63)+"1") // the rm of the job container (first id)
102 cancel := NewCanceller()
103 rec, res := ctrRunC(t, f, cancel, " container: ubuntu:24.04\n services:\n a:\n image: redis:7\n b:\n image: redis:7\n", " - run: echo ok\n", fastTimeouts)
104 if res.Conclusion == ConclusionSuccess {
105 t.Fatalf("a failed cleanup must be visible: %+v", res)
106 }
107 stop := logOf(rec.step(t, "Stop containers"))
108 for _, w := range []string{"Stop and remove container: ", "Print service container logs: ", "Remove container network: github_network_", "##[error]The docker command timed out after 400ms: rm --force ", "##[warning]Docker logs failed: The docker command timed out"} {
109 if !strings.Contains(stop, w) {
110 t.Errorf("stop lacks %q:\n%s", w, stop)
111 }
112 }
113 rms, nets := 0, 0
114 for _, c := range f.calls(t) {
115 if c[0] == "rm" {
116 rms++
117 }
118 if c[0] == "network" && len(c) > 1 && c[1] == "rm" {
119 nets++
120 }
121 }
122 // 3 removals, and the fallback of the job tries the one that timed out again.
123 if rms < 3 || rms > 4 || nets != 1 {
124 t.Errorf("rm=%d network rm=%d (want 3 or 4 and 1):\n%s", rms, nets, stop)
125 }
126 if rec.concl("Stop containers") != ConclusionFailure {
127 t.Errorf("Stop containers: %s", rec.concl("Stop containers"))
128 }
129}
130
131// GO-003: a shutdown skips the queued post steps (StepsRunner); the job-level
132// fallback removes the containers and the network all the same, and does not
133// run twice what "Stop containers" did.
134func TestShutdownStillCleansUpThroughTheFallback(t *testing.T) {
135 f := newFakeCtr(t)
136 f.set(t, "hang_exec", "1")
137 cancel := NewCanceller()
138 go func() { time.Sleep(600 * time.Millisecond); cancel.Cancel(CancelShutdown) }()
139 rec, res := ctrRunC(t, f, cancel, ctrJob, " - name: long\n run: sleep 60\n", nil)
140 _ = res
141 if rec.concl("Stop containers") == ConclusionSuccess {
142 t.Skip("the runner ran Stop containers on shutdown")
143 }
144 got := names(f.calls(t))
145 if !strings.HasSuffix(got, "rm network") {
146 t.Errorf("fallback did not remove the container and the network: %s", got)
147 }
148}
149
150func TestFallbackDoesNothingWhenStopContainersDidItAll(t *testing.T) {
151 f := newFakeCtr(t)
152 _, res := ctrRun(t, f, ctrJob, " - run: echo ok\n", nil)
153 if res.Conclusion != ConclusionSuccess {
154 t.Fatalf("%+v", res)
155 }
156 n := 0
157 for _, c := range f.calls(t) {
158 if c[0] == "rm" || (c[0] == "network" && len(c) > 1 && c[1] == "rm") {
159 n++
160 }
161 }
162 if n != 2 {
163 t.Errorf("%d removals, want exactly 2 (container, network): %s", n, names(f.calls(t)))
164 }
165}
166
167// GO-003: a panic of the job still cleans up.
168func TestPanicStillCleansUp(t *testing.T) {
169 f := newFakeCtr(t)
170 defer func() {
171 _ = recover()
172 got := names(f.calls(t))
173 if !strings.Contains(got, "rm") || !strings.HasSuffix(got, "rm network") {
174 t.Errorf("calls after the panic: %s", got)
175 }
176 }()
177 ctrRun(t, f, ctrJob, " - run: echo ok\n", func(c *Config) {
178 c.Reporter = &panicOnStep{Reporter: c.Reporter, name: "Run echo ok"}
179 })
180}
181
182type panicOnStep struct {
183 Reporter
184 name string
185}
186
187func (p *panicOnStep) StepStarted(n int, name string, at time.Time) {
188 if name == p.name {
189 panic("boom")
190 }
191 p.Reporter.StepStarted(n, name, at)
192}
193
194// GO-007: the credentials directory is removed even when the pull fails; a
195// removal that fails is reported and left to the fallback.
196func TestRegistryConfigRemovedAfterFailedPullAndRemovalFailureIsReported(t *testing.T) {
197 f := newFakeCtr(t)
198 f.set(t, "pull_fail", "9")
199 var temp string
200 rec, _ := ctrRun(t, f, " container:\n image: ghcr.io/o/i:1\n credentials:\n username: u\n password: pw-secret\n", " - run: echo\n", func(c *Config) { temp = c.Layout.Temp })
201 if m, _ := filepath.Glob(filepath.Join(temp, ".docker_*")); len(m) != 0 {
202 t.Errorf("config left after a failed pull: %v", m)
203 }
204 _ = rec
205
206 if os.Geteuid() == 0 {
207 t.Skip("root removes anything")
208 }
209 f2 := newFakeCtr(t)
210 f2.set(t, "lock_cfg", "1")
211 rec2, res2 := ctrRun(t, f2, " container:\n image: ghcr.io/o/i:1\n credentials:\n username: u\n password: pw-secret\n", " - run: echo\n", func(c *Config) { temp = c.Layout.Temp })
212 defer func() {
213 m, _ := filepath.Glob(filepath.Join(temp, ".docker_*", "sub"))
214 for _, d := range m {
215 _ = os.Chmod(d, 0o755)
216 }
217 }()
218 if res2.Conclusion != ConclusionFailure {
219 t.Fatalf("%+v", res2)
220 }
221 if got := logOf(rec2.step(t, "Initialize containers")); !strings.Contains(got, "##[error]Failed to remove directory containing Docker client credentials:") {
222 t.Errorf("init:\n%s", got)
223 }
224}
225
226// GO-001-R: the health inspection takes the cancellation too.
227func TestCancelDuringHealthInspection(t *testing.T) {
228 f := newFakeCtr(t)
229 f.set(t, "hang_inspect", "1")
230 cancel := NewCanceller()
231 go func() { time.Sleep(500 * time.Millisecond); cancel.Cancel(CancelUser) }()
232 start := time.Now()
233 rec, res := ctrRunC(t, f, cancel, " services:\n s:\n image: redis:7\n", " - run: echo never\n", func(c *Config) {
234 c.Actions.Containers.Command = time.Minute
235 })
236 if res.Conclusion != ConclusionCancelled || time.Since(start) > 15*time.Second {
237 t.Fatalf("%+v after %s %v", res, time.Since(start), rec.names())
238 }
239 var health bool
240 for _, c := range f.calls(t) {
241 if c[0] == "inspect" && strings.Contains(strings.Join(c, " "), "Healthcheck") {
242 health = true
243 }
244 }
245 if !health {
246 t.Errorf("the health inspection was not reached: %s", names(f.calls(t)))
247 }
248}
249
250// GO-002-R: a failed removal is not a removal. It stays pending and the job's
251// fallback tries again; "No such container" is a removal.
252func TestFailedRemovalStaysPendingAndAbsenceCounts(t *testing.T) {
253 count := func(f *fakeCtr, verb string) int {
254 n := 0
255 for _, c := range f.calls(t) {
256 if c[0] == verb || (verb == "network rm" && c[0] == "network" && len(c) > 1 && c[1] == "rm") {
257 n++
258 }
259 }
260 return n
261 }
262 f := newFakeCtr(t)
263 f.set(t, "rm_fail", "1")
264 f.set(t, "netrm_fail", "1")
265 rec, _ := ctrRun(t, f, ctrJob, " - run: echo ok\n", fastTimeouts)
266 if count(f, "rm") != 2 || count(f, "network rm") != 2 {
267 t.Errorf("rm=%d network rm=%d, want the fallback to retry both once", count(f, "rm"), count(f, "network rm"))
268 }
269 if got := logOf(rec.step(t, "Stop containers")); !strings.Contains(got, "Docker rm fail with exit code 1") || !strings.Contains(got, "Docker network rm fail with exit code 1") {
270 t.Errorf("warnings:\n%s", got)
271 }
272
273 f2 := newFakeCtr(t)
274 f2.set(t, "rm_absent", "1")
275 f2.set(t, "netrm_absent", "1")
276 rec2, res2 := ctrRun(t, f2, ctrJob, " - run: echo ok\n", fastTimeouts)
277 if res2.Conclusion != ConclusionSuccess || count(f2, "rm") != 1 || count(f2, "network rm") != 1 {
278 t.Errorf("absent: %+v rm=%d network rm=%d\n%s", res2, count(f2, "rm"), count(f2, "network rm"), logOf(rec2.step(t, "Stop containers")))
279 }
280}
281
282// GO-007-R: the credentials directory is registered before the login; a panic
283// in the login or in the pull leaves nothing behind.
284func TestPanicDuringLoginOrPullLeavesNoCredentials(t *testing.T) {
285 for _, during := range []string{" login ", " pull "} {
286 f := newFakeCtr(t)
287 var temp string
288 func() {
289 defer func() { _ = recover() }()
290 ctrRun(t, f, " container:\n image: ghcr.io/o/i:1\n credentials:\n username: u\n password: pw-secret\n", " - run: echo\n", func(c *Config) {
291 temp = c.Layout.Temp
292 c.Reporter = &panicOnLog{Reporter: c.Reporter, contains: during}
293 })
294 }()
295 if m, _ := filepath.Glob(filepath.Join(temp, ".docker_*")); len(m) != 0 {
296 t.Errorf("panic during%s: credentials left: %v", during, m)
297 }
298 }
299}
300
301type panicOnLog struct {
302 Reporter
303 contains string
304}
305
306func (p *panicOnLog) Log(n int, at time.Time, line string) {
307 if strings.HasPrefix(line, "##[command]") && strings.Contains(line, p.contains) {
308 panic("boom")
309 }
310 p.Reporter.Log(n, at, line)
311}
312
313// GO-004-R3: a variable name that holds "=" or NUL, or is empty, is refused
314// for create and exec, before docker is called.
315func TestInvalidEnvNamesAreRefused(t *testing.T) {
316 if checkEnvNames([]string{"A", "b_c", "A.B", "A-B"}) != nil {
317 t.Error("valid names refused")
318 }
319 for _, k := range []string{"", "A=B", "HTTP_PROXY=http://x/?y", "A\x00B", "="} {
320 if err := checkEnvNames([]string{"OK", k}); err == nil || !strings.Contains(err.Error(), "Invalid environment variable name") {
321 t.Errorf("%q: %v", k, err)
322 }
323 }
324 f := newFakeCtr(t)
325 rec, res := ctrRun(t, f, " container:\n image: ubuntu:24.04\n env:\n \"HTTP_PROXY=http://x/?y\": v\n", " - run: echo never\n", nil)
326 if res.Conclusion != ConclusionFailure || !strings.Contains(logOf(rec.step(t, "Set up job")), "##[error]Invalid environment variable name 'HTTP_PROXY=http://x/?y'") {
327 t.Errorf("container env: %+v", res)
328 }
329 rec, res = ctrRun(t, f, ctrJob, " - name: s\n env:\n \"HTTP_PROXY=http://x/?y\": v\n run: echo never\n", nil)
330 if res.Conclusion != ConclusionFailure || !strings.Contains(logOf(rec.step(t, "s")), "##[error]Invalid environment variable name 'HTTP_PROXY=http://x/?y'") {
331 t.Errorf("step env: %+v\n%s", res, logOf(rec.step(t, "s")))
332 }
333 if n := len(f.execCalls(t)); n != 0 {
334 t.Errorf("docker exec called %d times", n)
335 }
336}
337
338// GO-008: the budget of the whole cleanup is checked before every command. With
339// a budget spent by the first stalled removal, the others (and the retry of the
340// fallback) are not started: they would each cost a signal escalation.
341func TestCleanupBudgetIsCheckedBeforeEveryCommand(t *testing.T) {
342 f := newFakeCtr(t)
343 f.set(t, "hang_for", "rm --force")
344 f.set(t, "hang_ignore", "1") // SIGINT is ignored: a started command costs TermAfter
345 svcs := ""
346 for _, n := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {
347 svcs += " " + n + ":\n image: redis:7\n"
348 }
349 start := time.Now()
350 rec, res := ctrRun(t, f, " container: ubuntu:24.04\n services:\n"+svcs, " - run: echo ok\n", func(c *Config) {
351 c.Actions.Containers = ContainerTimeouts{Command: time.Minute, Pull: time.Minute, CleanupCommand: time.Minute, CleanupBudget: time.Second}
352 })
353 total := time.Since(start)
354 if res.Conclusion == ConclusionSuccess {
355 t.Fatalf("an unfinished cleanup must be visible: %+v", res)
356 }
357 stop := rec.step(t, "Stop containers")
358 took := stop.Completed.Sub(stop.Started)
359 // The budget (1 s) and one escalation (TermAfter 200 ms in the tests, plus the kill delay).
360 if took > 1900*time.Millisecond || total > 12*time.Second {
361 t.Errorf("Stop containers took %s (run %s), want about the budget plus one escalation", took, total)
362 }
363 rms := 0
364 for _, c := range f.calls(t) {
365 if c[0] == "rm" {
366 rms++
367 }
368 }
369 if rms != 1 {
370 t.Errorf("%d removals started, want 1 (the budget was spent by it)", rms)
371 }
372 log := logOf(stop)
373 if !strings.Contains(log, "##[error]The cleanup of the containers ran out of time (1s): stopped: docker rm --force ") ||
374 !strings.Contains(log, "ran out of time (1s): not run: docker ") {
375 t.Errorf("log:\n%s", log)
376 }
377 if strings.Count(log, "not run: docker network rm") != 1 {
378 t.Errorf("the network removal must be reported as not run once:\n%s", log)
379 }
380}