1// SPDX-License-Identifier: MIT
2
3package executor
4
5import (
6 "fmt"
7 "math/rand/v2"
8 "os"
9 "path/filepath"
10 "regexp"
11 "slices"
12 "strconv"
13 "strings"
14 "sync"
15 "time"
16)
17
18// containerSet is the containers of a job: the job container, if any, then the
19// services (JobExtension.cs:322-326: the order of the list the provider gets).
20type containerSet struct {
21 job *container
22 services []*container
23 network string
24
25 // What the cleanup needs, recorded BEFORE the command that creates it, so
26 // that a command whose outcome is not known (a timeout, a cancellation, a
27 // lost connection) can still be cleaned: the names are chosen here, not by
28 // docker. mu guards them; cleanupMu serialises the cleanups ("Stop
29 // containers" and the fallback of the job).
30 mu sync.Mutex
31 cleanupMu sync.Mutex
32 networkTry string // network name, set before docker network create
33 netRemoved bool // the network is gone
34 dirs []string // docker --config directories that could not be removed
35 // deadline is the end of the cleanup budget, fixed by the first cleanup of
36 // the job: "Stop containers" and the fallback share it, so the cleanup of a
37 // job is bounded as a whole.
38 deadline time.Time
39}
40
41func (cs *containerSet) tryNetwork(name string) { cs.mu.Lock(); cs.networkTry = name; cs.mu.Unlock() }
42
43func (cs *containerSet) addDir(d string) {
44 cs.mu.Lock()
45 defer cs.mu.Unlock()
46 if !slices.Contains(cs.dirs, d) {
47 cs.dirs = append(cs.dirs, d)
48 }
49}
50
51// delDir forgets a directory that is gone.
52func (cs *containerSet) delDir(d string) {
53 cs.mu.Lock()
54 cs.dirs = slices.DeleteFunc(cs.dirs, func(x string) bool { return x == d })
55 cs.mu.Unlock()
56}
57
58func (cs *containerSet) all() []*container {
59 var l []*container
60 if cs.job != nil {
61 l = append(l, cs.job)
62 }
63 return append(l, cs.services...)
64}
65
66// minDockerAPI is requiredDockerEngineAPIVersion (ContainerOperationProvider.cs
67// :591): Docker-CE 17.12.
68var minDockerAPI = [2]int{1, 35}
69
70var (
71 apiVersionRe = regexp.MustCompile(`(?i)\d+\.\d+(\.\d+)?`)
72 portLineRe = regexp.MustCompile(`^(\d+)/(\w+) -> (.+):(\d+)$`)
73)
74
75// initContainersStep is the "Initialize containers" step
76// (JobExtension.cs:328-331), run if success().
77func (j *job) initContainersStep() qstep {
78 return qstep{name: "Initialize containers", setup: func(s *stepRun, cancel <-chan struct{}) stepResult {
79 return j.initContainers(s, cancel)
80 }}
81}
82
83// result turns the error of a step body into the result StepsRunner gives it
84// (StepsRunner.cs:314-340): a cancellation is logged and cancels the step,
85// anything else is logged with its message and fails it.
86func (j *job) containerResult(s *stepRun, err error) stepResult {
87 if err == nil {
88 return resSucceeded
89 }
90 if err == errCancelled {
91 return j.cancelledStep(s)
92 }
93 j.errorf(s.num, "%s", err.Error())
94 return resFailed
95}
96
97// initContainers is ContainerOperationProvider.StartContainersAsync
98// (:40-103). The step that stops the containers is registered first, so that
99// it runs even when this one fails (:56-60).
100func (j *job) initContainers(s *stepRun, cancel <-chan struct{}) stepResult {
101 j.debug(s.num, "Register post job cleanup for stopping/deleting containers.")
102 j.mu.Lock()
103 j.postStack = append(j.postStack, qstep{name: "Stop containers", cond: "always()", setup: j.stopContainers})
104 j.mu.Unlock()
105 return j.containerResult(s, j.startContainers(s, cancel))
106}
107
108func (j *job) startContainers(s *stepRun, cancel <-chan struct{}) error {
109 num, cs := s.num, j.cs
110 // AssertCompatibleOS (:559-571) refuses to run in a container because the
111 // paths it mounts would be those of the container's host. The docker
112 // daemon of a job runs next to the executor, in the same sandbox, so the
113 // paths are right and the check is left out.
114 if err := j.checkDockerVersion(s, cancel); err != nil {
115 return err
116 }
117 j.out(num, "##[group]Clean up resources from previous jobs")
118 label := j.dockerLabel()
119 ps := j.newDockerCmd("ps", raw("--all", "--quiet", "--no-trunc"), quotedOpt("--filter", "label="+label))
120 stale, err := j.dockerList(s, ps, cancel)
121 if err != nil {
122 return err
123 }
124 for _, id := range stale {
125 code, err := j.dockerCode(s, j.newDockerCmd("rm", raw("--force", id)), cancel)
126 if err != nil {
127 return err
128 }
129 if code != 0 {
130 j.tagged(num, "##[warning]", fmt.Sprintf("Delete stale containers failed, docker rm fail with exit code %d for container %s", code, id))
131 }
132 }
133 prune := j.newDockerCmd("network", raw("prune", "--force"), quotedOpt("--filter", "label="+label))
134 code, err := j.dockerCode(s, prune, cancel)
135 if err != nil {
136 return err
137 }
138 if code != 0 {
139 j.tagged(num, "##[warning]", fmt.Sprintf("Delete stale container networks failed, docker network prune fail with exit code %d", code))
140 }
141 j.out(num, "##[endgroup]")
142
143 j.out(num, "##[group]Create local container network")
144 network := "github_network_" + strings.ReplaceAll(newImageTag(""), ":", "")
145 cs.tryNetwork(network)
146 code, err = j.dockerCode(s, j.newDockerCmd("network", raw("create", "--label", label, network)), cancel)
147 if err != nil {
148 return err
149 }
150 if code != 0 {
151 return ghErrorf("Docker network create failed with exit code %d", code)
152 }
153 j.mu.Lock()
154 j.state.SetJobContainer("network", network)
155 j.mu.Unlock()
156 cs.network = network
157 j.out(num, "##[endgroup]")
158
159 for _, c := range cs.all() {
160 c.network = network
161 if err := j.startContainer(s, c, cancel); err != nil {
162 return err
163 }
164 }
165 return j.healthchecks(s, cs.all(), cancel)
166}
167
168// checkDockerVersion is AssertCompatibleOS' version check (:573-598) with
169// DockerCommandManager.DockerVersion (:47-90), including its use of the
170// server's text for the client version (:71, the runner's own slip).
171func (j *job) checkDockerVersion(s *stepRun, cancel <-chan struct{}) error {
172 num := s.num
173 j.out(num, "##[group]Checking docker version")
174 ver := func(which string) (string, error) {
175 l, err := j.dockerList(s, j.newDockerCmd("version", raw("--format", "'{{."+which+".APIVersion}}'")), cancel)
176 if err != nil {
177 return "", err
178 }
179 if len(l) == 0 {
180 return "", ghErrorf("Value cannot be null. (Parameter 'Docker.%s.Version')", which)
181 }
182 return l[0], nil
183 }
184 server, err := ver("Server")
185 if err != nil {
186 return err
187 }
188 j.out(num, "Docker daemon API version: "+server)
189 client, err := ver("Client")
190 if err != nil {
191 return err
192 }
193 j.out(num, "Docker client API version: "+client)
194 j.out(num, "##[endgroup]")
195 v, ok := parseAPIVersion(server)
196 if !ok {
197 return ghErrorf("Value cannot be null. (Parameter 'ServerVersion')")
198 }
199 if v[0] < minDockerAPI[0] || (v[0] == minDockerAPI[0] && v[1] < minDockerAPI[1]) {
200 return ghErrorf("Min required docker engine API server version is '%d.%d', your docker ('%s') server version is '%d.%d'",
201 minDockerAPI[0], minDockerAPI[1], j.dockerPath(), v[0], v[1])
202 }
203 return nil
204}
205
206func parseAPIVersion(s string) ([2]int, bool) {
207 m := apiVersionRe.FindString(s)
208 parts := strings.Split(m, ".")
209 if len(parts) < 2 {
210 return [2]int{}, false
211 }
212 a, e1 := strconv.Atoi(parts[0])
213 b, e2 := strconv.Atoi(parts[1])
214 return [2]int{a, b}, e1 == nil && e2 == nil
215}
216
217// containerBackoff is BackoffTimerHelper.GetRandomBackoff(1 s, 10 s).
218func (j *job) containerBackoff() time.Duration {
219 if f := j.cfg.Actions.Backoff; f != nil {
220 return f()
221 }
222 return time.Duration(1000+rand.IntN(9000)) * time.Millisecond
223}
224
225// seconds is the way the runner prints a backoff, TimeSpan.TotalSeconds.
226func seconds(d time.Duration) string { return strconv.FormatFloat(d.Seconds(), 'f', -1, 64) }
227
228// pullArgs builds `docker [--config dir] pull image`.
229func (j *job) pullCmd(image, config string) dockerCmd {
230 if config == "" {
231 return j.newDockerCmd("pull", raw(image))
232 }
233 p := j.dockerPath()
234 return dockerCmd{line: p + " --config " + config + " pull " + image, argv: []string{p, "--config", config, "pull", image}}
235}
236
237// removeConfigDir is ContainerRegistryLogout (:457-470): the credentials
238// directory goes, and a failure is the runner's error. A directory that
239// cannot be removed is kept for the cleanup of the job, which tries again.
240func (j *job) removeConfigDir(dir string) error {
241 if dir == "" {
242 return nil
243 }
244 if err := os.RemoveAll(dir); err != nil {
245 j.cs.addDir(dir)
246 return ghErrorf("Failed to remove directory containing Docker client credentials: %v", err)
247 }
248 j.cs.delDir(dir) // unregistered only once it is gone
249 return nil
250}
251
252// registryLogin is ContainerRegistryLogin (:394-455): the credentials go to a
253// private docker config directory, the password through stdin. It returns the
254// directory ("" without credentials); the caller removes it with
255// removeConfigDir, whatever happens.
256func (j *job) registryLogin(s *stepRun, c *container, cancel <-chan struct{}) (string, error) {
257 if c.credUser == "" || c.credPass == "" {
258 return "", nil
259 }
260 // The password is a secret for the logs of this job from here on.
261 j.addMask(c.credPass)
262 dir := filepath.Join(j.state.Layout().Temp, ".docker_"+strings.ReplaceAll(newImageTag(""), ":", ""))
263 if err := os.MkdirAll(dir, 0o700); err != nil {
264 return "", ghErrorf("Failed to create directory to store registry client credentials: %v", err)
265 }
266 // Registered with the cleanup of the job before anything can fail or panic:
267 // from here on the fallback removes it whatever happens.
268 j.cs.addDir(dir)
269 j.own(dir)
270 args := []string{j.dockerPath(), "--config", dir, "login"}
271 line := "--config " + dir + " login"
272 if c.registryServer != "" {
273 args, line = append(args, c.registryServer), line+" "+c.registryServer
274 }
275 args, line = append(args, "-u", c.credUser, "--password-stdin"), line+" -u "+c.credUser+" --password-stdin"
276 for try := 1; ; try++ {
277 j.out(s.num, "##[command]"+j.dockerPath()+" "+line)
278 bc, timedOut, stop := bound(cancel, j.timeouts().Command)
279 res, err := runProcess(procSpec{
280 Argv: args, Dir: dirOrCwd(j.githubValue("workspace")), Env: j.processEnv(newOrdered()), Credential: j.cfg.Credential,
281 OnLine: func(string) {}, Stdin: c.credPass + "\n",
282 Cancel: bc, TermAfter: j.cfg.TermAfter, KillAfter: j.cfg.KillAfter, WaitDelay: j.cfg.WaitDelay, Signal: j.cfg.Signal,
283 })
284 stop()
285 if err != nil {
286 return dir, err
287 }
288 if res.Cancelled {
289 if timedOut() {
290 return dir, &cmdTimeoutError{cmd: "login", d: j.timeouts().Command}
291 }
292 return dir, errCancelled
293 }
294 if res.ExitCode == 0 {
295 return dir, nil
296 }
297 if try == 3 {
298 return dir, ghErrorf("Docker login for '%s' failed with exit code %d", c.registryServer, res.ExitCode)
299 }
300 d := j.containerBackoff()
301 j.tagged(s.num, "##[warning]", fmt.Sprintf("Docker login for '%s' failed with exit code %d, back off %s seconds before retry.", c.registryServer, res.ExitCode, seconds(d)))
302 if !j.sleepOrCancel(d, cancel) {
303 return dir, errCancelled
304 }
305 }
306}