1// SPDX-License-Identifier: MIT
2
3package container
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "net/netip"
11 "strings"
12
13 cerrdefs "github.com/containerd/errdefs"
14 "github.com/moby/moby/api/types/container"
15 "github.com/moby/moby/api/types/mount"
16 "github.com/moby/moby/api/types/network"
17 "github.com/moby/moby/api/types/system"
18 "github.com/moby/moby/client"
19)
20
21// engine is the part of the Docker Engine API the driver uses. The real
22// implementation wraps github.com/moby/moby/client (the maintained client of
23// the Moby project; github.com/docker/docker/client is its frozen
24// predecessor); tests use a fake.
25type engine interface {
26 Info(ctx context.Context) (daemonInfo, error)
27 // EnsureImage returns nil when ref exists locally, pulling it otherwise.
28 EnsureImage(ctx context.Context, ref string) error
29 CreateNetwork(ctx context.Context, spec networkSpec) (string, error)
30 RemoveNetwork(ctx context.Context, id string) error
31 ListNetworks(ctx context.Context, labelKey string) ([]string, error)
32 CreateContainer(ctx context.Context, spec containerSpec) (string, error)
33 StartContainer(ctx context.Context, id string) error
34 // WaitExit returns a channel that yields the exit code once the container
35 // stops running (or -1 and an error when the wait itself fails).
36 WaitExit(ctx context.Context, id string) <-chan exitResult
37 RemoveContainer(ctx context.Context, id string) error
38 ListContainers(ctx context.Context, labelKey string) ([]string, error)
39 LogsTail(ctx context.Context, id string, lines int) (string, error)
40 Close() error
41}
42
43type exitResult struct {
44 Code int
45 Err error
46}
47
48type daemonInfo struct {
49 Rootless bool
50 Runtimes []string
51 OSType string
52 // Podman: the "Docker API" is podman's compatibility service, whose netavark
53 // configures a bridge address only when the first container starts.
54 Podman bool
55}
56
57type networkSpec struct {
58 Name string
59 Bridge string // host bridge device name
60 Subnet netip.Prefix
61 Gateway netip.Addr
62 Labels map[string]string
63}
64
65type containerSpec struct {
66 Name string
67 Image string
68 Entrypoint []string
69 Env []string
70 Labels map[string]string
71 Runtime string // "" = daemon default
72 Privileged bool
73 Network string
74 IPv4 netip.Addr
75 DNS netip.Addr
76 Binds []bindMount
77 MemoryMiB int // also the swap limit: no swap
78 CPUs int
79 Pids int // PidsLimit
80}
81
82type bindMount struct {
83 Source, Target string
84 ReadOnly bool
85}
86
87// dockerEngine implements engine with the moby client.
88type dockerEngine struct{ cli *client.Client }
89
90// newDockerEngine connects like the docker CLI: DOCKER_HOST, DOCKER_TLS_VERIFY,
91// DOCKER_CERT_PATH, DOCKER_API_VERSION, with API version negotiation.
92func newDockerEngine() (*dockerEngine, error) {
93 cli, err := client.New(client.FromEnv)
94 if err != nil {
95 return nil, fmt.Errorf("docker client: %w", err)
96 }
97 return &dockerEngine{cli: cli}, nil
98}
99
100func (e *dockerEngine) Close() error { return e.cli.Close() }
101
102func (e *dockerEngine) Info(ctx context.Context) (daemonInfo, error) {
103 r, err := e.cli.Info(ctx, client.InfoOptions{})
104 if err != nil {
105 return daemonInfo{}, fmt.Errorf("docker info: %w", err)
106 }
107 di := daemonInfo{OSType: r.Info.OSType}
108 for _, o := range r.Info.SecurityOptions {
109 if o == "name=rootless" {
110 di.Rootless = true
111 }
112 }
113 for name := range r.Info.Runtimes {
114 di.Runtimes = append(di.Runtimes, name)
115 }
116 // Best effort: a daemon that cannot say what it is is treated as Docker.
117 if v, err := e.cli.ServerVersion(ctx, client.ServerVersionOptions{}); err == nil {
118 di.Podman = isPodman(v.Components)
119 }
120 return di, nil
121}
122
123// isPodman reports whether the daemon's version components name Podman (its
124// compatibility service lists "Podman Engine"; Docker lists "Engine").
125func isPodman(components []system.ComponentVersion) bool {
126 for _, c := range components {
127 if strings.HasPrefix(c.Name, "Podman") {
128 return true
129 }
130 }
131 return false
132}
133
134func (e *dockerEngine) EnsureImage(ctx context.Context, ref string) error {
135 if _, err := e.cli.ImageInspect(ctx, ref); err == nil {
136 return nil
137 } else if !cerrdefs.IsNotFound(err) {
138 return fmt.Errorf("inspect image %s: %w", ref, err)
139 }
140 rc, err := e.cli.ImagePull(ctx, ref, client.ImagePullOptions{})
141 if err != nil {
142 return fmt.Errorf("image %s is not present and cannot be pulled (build it with scripts/build-image.sh): %w", ref, err)
143 }
144 defer func() { _ = rc.Close() }()
145 if err := rc.Wait(ctx); err != nil {
146 return fmt.Errorf("pull image %s: %w", ref, err)
147 }
148 return nil
149}
150
151func (e *dockerEngine) CreateNetwork(ctx context.Context, s networkSpec) (string, error) {
152 f := false
153 r, err := e.cli.NetworkCreate(ctx, s.Name, client.NetworkCreateOptions{
154 Driver: "bridge",
155 EnableIPv6: &f,
156 IPAM: &network.IPAM{Config: []network.IPAMConfig{{
157 Subnet: s.Subnet, Gateway: s.Gateway,
158 }}},
159 Options: map[string]string{
160 "com.docker.network.bridge.name": s.Bridge,
161 // The runner's nftables table does the NAT, with its own policy.
162 "com.docker.network.bridge.enable_ip_masquerade": "false",
163 },
164 Labels: s.Labels,
165 })
166 if err != nil {
167 return "", fmt.Errorf("create network %s: %w", s.Name, err)
168 }
169 return r.ID, nil
170}
171
172func (e *dockerEngine) RemoveNetwork(ctx context.Context, id string) error {
173 _, err := e.cli.NetworkRemove(ctx, id, client.NetworkRemoveOptions{})
174 return ignoreNotFound(err)
175}
176
177func (e *dockerEngine) ListNetworks(ctx context.Context, key string) ([]string, error) {
178 r, err := e.cli.NetworkList(ctx, client.NetworkListOptions{Filters: client.Filters{}.Add("label", key)})
179 if err != nil {
180 return nil, err
181 }
182 var ids []string
183 for _, n := range r.Items {
184 ids = append(ids, n.ID)
185 }
186 return ids, nil
187}
188
189// hostConfigFor builds the Docker host configuration of a job container.
190func hostConfigFor(s containerSpec) *container.HostConfig {
191 hc := &container.HostConfig{
192 NetworkMode: container.NetworkMode(s.Network),
193 Privileged: s.Privileged,
194 Runtime: s.Runtime,
195 CgroupnsMode: container.CgroupnsModePrivate,
196 DNS: []netip.Addr{s.DNS},
197 Sysctls: map[string]string{"net.ipv6.conf.all.disable_ipv6": "1", "net.ipv6.conf.default.disable_ipv6": "1"},
198 }
199 if s.MemoryMiB > 0 {
200 hc.Memory = int64(s.MemoryMiB) << 20
201 hc.MemorySwap = hc.Memory // swap = memory: the job cannot use swap
202 }
203 if s.CPUs > 0 {
204 hc.NanoCPUs = int64(s.CPUs) * 1_000_000_000
205 }
206 if s.Pids > 0 {
207 p := int64(s.Pids)
208 hc.PidsLimit = &p
209 }
210 for _, b := range s.Binds {
211 hc.Mounts = append(hc.Mounts, mount.Mount{Type: mount.TypeBind, Source: b.Source, Target: b.Target, ReadOnly: b.ReadOnly,
212 BindOptions: &mount.BindOptions{CreateMountpoint: true}})
213 }
214 return hc
215}
216
217func (e *dockerEngine) CreateContainer(ctx context.Context, s containerSpec) (string, error) {
218 hc := hostConfigFor(s)
219 cfg := &container.Config{
220 Image: s.Image,
221 Entrypoint: s.Entrypoint,
222 Cmd: []string{},
223 Env: s.Env,
224 Labels: s.Labels,
225 Hostname: "ermite-job",
226 }
227 nc := &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{
228 s.Network: {IPAMConfig: &network.EndpointIPAMConfig{IPv4Address: s.IPv4}},
229 }}
230 r, err := e.cli.ContainerCreate(ctx, client.ContainerCreateOptions{Name: s.Name, Config: cfg, HostConfig: hc, NetworkingConfig: nc})
231 if err != nil {
232 return "", fmt.Errorf("create container: %w", err)
233 }
234 return r.ID, nil
235}
236
237func (e *dockerEngine) StartContainer(ctx context.Context, id string) error {
238 _, err := e.cli.ContainerStart(ctx, id, client.ContainerStartOptions{})
239 return err
240}
241
242func (e *dockerEngine) WaitExit(ctx context.Context, id string) <-chan exitResult {
243 w := e.cli.ContainerWait(ctx, id, client.ContainerWaitOptions{Condition: container.WaitConditionNextExit})
244 return awaitExit(ctx, w.Result, w.Error)
245}
246
247// awaitExit turns the two channels of a Docker wait into one result. A context
248// that ended is reported as such (code -1, ctx.Err()), whichever channel the
249// select picked: a cancelled wait must never look like a clean exit.
250func awaitExit(ctx context.Context, result <-chan container.WaitResponse, errc <-chan error) <-chan exitResult {
251 out := make(chan exitResult, 1)
252 go func() {
253 var res exitResult
254 select {
255 case r := <-result:
256 res = exitResult{Code: int(r.StatusCode)}
257 case err := <-errc:
258 res = exitResult{Code: -1, Err: err}
259 case <-ctx.Done():
260 res = exitResult{Code: -1, Err: ctx.Err()}
261 }
262 if err := ctx.Err(); err != nil {
263 res = exitResult{Code: -1, Err: err}
264 }
265 out <- res
266 }()
267 return out
268}
269
270func (e *dockerEngine) RemoveContainer(ctx context.Context, id string) error {
271 _, err := e.cli.ContainerRemove(ctx, id, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
272 return ignoreNotFound(err)
273}
274
275func (e *dockerEngine) ListContainers(ctx context.Context, key string) ([]string, error) {
276 r, err := e.cli.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: client.Filters{}.Add("label", key)})
277 if err != nil {
278 return nil, err
279 }
280 var ids []string
281 for _, c := range r.Items {
282 ids = append(ids, c.ID)
283 }
284 return ids, nil
285}
286
287func (e *dockerEngine) LogsTail(ctx context.Context, id string, lines int) (string, error) {
288 rc, err := e.cli.ContainerLogs(ctx, id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: fmt.Sprint(lines)})
289 if err != nil {
290 return "", err
291 }
292 defer func() { _ = rc.Close() }()
293 b, err := io.ReadAll(io.LimitReader(rc, 64<<10))
294 return string(b), err
295}
296
297func ignoreNotFound(err error) error {
298 if err == nil || cerrdefs.IsNotFound(err) {
299 return nil
300 }
301 return err
302}
303
304var errRootless = errors.New("the Docker daemon is rootless: a job needs its own nested dockerd, which requires a rootful daemon (or the sysbox runtime); point DOCKER_HOST at a rootful daemon or use ERMITE_RUNNER_ISOLATION=vm")