1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "archive/zip"
7 "context"
8 "crypto/rand"
9 "encoding/hex"
10 "errors"
11 "fmt"
12 "io"
13 "io/fs"
14 "os"
15 "path"
16 "path/filepath"
17 "slices"
18 "strconv"
19 "strings"
20
21 "connectrpc.com/connect"
22 "github.com/spf13/cobra"
23
24 "bnema.dev/ermite-api/gen/ermite"
25 "bnema.dev/ermite/internal/adapters/client"
26)
27
28// artifactZip is the media type of an artifact that is a zip archive.
29const artifactZip = "application/zip"
30
31// maxExtractBytes bounds what the extraction of one archive writes: a small
32// archive can declare a huge content.
33var maxExtractBytes int64 = 32 << 30
34
35// maxZipEntries bounds the entries of one archive.
36const maxZipEntries = 100_000
37
38// jsonArtifact is the stable JSON shape of an artifact.
39type jsonArtifact struct {
40 ID int64 `json:"id"`
41 Name string `json:"name"`
42 Size int64 `json:"size"`
43 Digest string `json:"digest"`
44 MediaType string `json:"media_type"`
45 RunID int64 `json:"run_id"`
46 Attempt int32 `json:"attempt"`
47 JobID string `json:"job_id"`
48 CreatedAt *string `json:"created_at"`
49 ExpiresAt *string `json:"expires_at"`
50}
51
52type jsonArtifactList struct {
53 Artifacts []jsonArtifact `json:"artifacts"`
54 UsedBytes int64 `json:"used_bytes"`
55 QuotaBytes int64 `json:"quota_bytes"`
56}
57
58func toJSONArtifact(a *ermite.Artifact) jsonArtifact {
59 return jsonArtifact{
60 ID: a.GetId(), Name: a.GetName(), Size: a.GetSize(), Digest: a.GetDigest(), MediaType: a.GetMediaType(),
61 RunID: a.GetRunId(), Attempt: a.GetAttempt(), JobID: a.GetJobId(),
62 CreatedAt: jsonTime(a.GetCreatedAt()), ExpiresAt: jsonTime(a.GetExpiresAt()),
63 }
64}
65
66// humanBytes is a size for people, in binary units.
67func humanBytes(n int64) string {
68 const unit = 1024
69 if n < unit {
70 return strconv.FormatInt(n, 10) + " B"
71 }
72 div, exp := int64(unit), 0
73 for m := n / unit; m >= unit && exp < 4; m /= unit {
74 div *= unit
75 exp++
76 }
77 return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTP"[exp])
78}
79
80// listRunArtifacts reads the artifacts of a run. A server without them
81// answers unimplemented, said in words here.
82func listRunArtifacts(ctx context.Context, c *client.Client, repo string, runID int64) (*ermite.ListArtifactsResponse, error) {
83 res, err := c.Artifacts().ListArtifacts(ctx, connect.NewRequest(&ermite.ListArtifactsRequest{Repo: repo, RunId: runID}))
84 if connect.CodeOf(err) == connect.CodeUnimplemented {
85 return nil, fmt.Errorf("the server does not keep artifacts (it lacks the ci.artifacts feature)")
86 }
87 if err != nil {
88 return nil, err
89 }
90 return res.Msg, nil
91}
92
93func newRunArtifactsCmd(env *cliEnv) *cobra.Command {
94 cmd := &cobra.Command{
95 Use: "artifacts <repo> <run id>",
96 Short: "List the artifacts of a run",
97 Long: "List the artifacts that the jobs of a run uploaded (actions/upload-artifact), with the space the\n" +
98 "artifacts of the repository use. Needs a token that reads the repository, even a public one.\n" +
99 "JSON fields (--json): artifacts (id, name, size, digest, media_type, run_id, attempt, job_id,\n" +
100 "created_at, expires_at), used_bytes and quota_bytes.",
101 Args: cobra.ExactArgs(2),
102 ValidArgsFunction: completeRepoNames(env),
103 }
104 asJSON := addJSONFlag(cmd)
105 cmd.RunE = func(cmd *cobra.Command, args []string) error {
106 id, err := parseRunID(args[1])
107 if err != nil {
108 return err
109 }
110 c, err := env.client(cmd)
111 if err != nil {
112 return err
113 }
114 res, err := listRunArtifacts(env.ctx(cmd), c, args[0], id)
115 if err != nil {
116 return err
117 }
118 if *asJSON {
119 out := jsonArtifactList{Artifacts: []jsonArtifact{}, UsedBytes: res.GetUsedBytes(), QuotaBytes: res.GetQuotaBytes()}
120 for _, a := range res.GetArtifacts() {
121 out.Artifacts = append(out.Artifacts, toJSONArtifact(a))
122 }
123 return writeJSON(cmd.OutOrStdout(), out)
124 }
125 w := cmd.OutOrStdout()
126 if len(res.GetArtifacts()) == 0 {
127 noticef(cmd, "run %d has no artifacts", id)
128 } else {
129 rows := make([][]string, 0, len(res.GetArtifacts()))
130 for _, a := range res.GetArtifacts() {
131 rows = append(rows, []string{
132 strconv.FormatInt(a.GetId(), 10), a.GetName(), humanBytes(a.GetSize()), shortTime(a.GetExpiresAt(), "-"),
133 })
134 }
135 if err := table(w, []string{"ID", "NAME", "SIZE", "EXPIRES"}, rows); err != nil {
136 return err
137 }
138 }
139 _, _ = fmt.Fprintln(w, quotaLine(res.GetUsedBytes(), res.GetQuotaBytes()))
140 return nil
141 }
142 return cmd
143}
144
145// quotaLine says how much of its space the artifacts of the repository use.
146func quotaLine(used, quota int64) string {
147 if quota <= 0 {
148 return "artifacts of the repository: " + humanBytes(used) + " used"
149 }
150 return "artifacts of the repository: " + humanBytes(used) + " of " + humanBytes(quota) + " used"
151}
152
153func newRunDownloadCmd(env *cliEnv) *cobra.Command {
154 var names []string
155 var dir string
156 cmd := &cobra.Command{
157 Use: "download <repo> <run id>",
158 Short: "Download the artifacts of a run",
159 Long: "Download the artifacts of a run, all of them or those that --name chooses (repeat it for several).\n" +
160 "A zip artifact is extracted: into --dir when one artifact is downloaded, into <dir>/<name> when\n" +
161 "several are. An artifact that is not a zip (uploaded with archive: false) is saved as <dir>/<name>.\n" +
162 "Existing files are never overwritten. Needs a token that reads the repository, even a public one.",
163 Args: cobra.ExactArgs(2),
164 ValidArgsFunction: completeRepoNames(env),
165 }
166 cmd.Flags().StringArrayVarP(&names, "name", "n", nil, "download the artifact of this name (repeatable)")
167 cmd.Flags().StringVarP(&dir, "dir", "D", ".", "the directory to download into")
168 cmd.RunE = func(cmd *cobra.Command, args []string) error {
169 id, err := parseRunID(args[1])
170 if err != nil {
171 return err
172 }
173 c, err := env.client(cmd)
174 if err != nil {
175 return err
176 }
177 ctx := env.ctx(cmd)
178 res, err := listRunArtifacts(ctx, c, args[0], id)
179 if err != nil {
180 return err
181 }
182 picked, err := pickArtifacts(res.GetArtifacts(), names)
183 if err != nil {
184 return err
185 }
186 if len(picked) == 0 {
187 return fmt.Errorf("run %d has no artifacts", id)
188 }
189 for _, a := range picked {
190 if err := checkArtifactName(a.GetName()); err != nil {
191 return err
192 }
193 }
194 if err := os.MkdirAll(dir, 0o755); err != nil {
195 return err
196 }
197 root, err := os.OpenRoot(dir)
198 if err != nil {
199 return err
200 }
201 defer func() { _ = root.Close() }()
202 for _, a := range picked {
203 to, err := downloadArtifact(ctx, c, args[0], a, root, len(picked) > 1)
204 if err != nil {
205 return fmt.Errorf("artifact %q: %w", clean(a.GetName()), err)
206 }
207 noticef(cmd, "downloaded %s to %s", clean(a.GetName()), clean(filepath.Join(dir, to)))
208 }
209 return nil
210 }
211 return cmd
212}
213
214// pickArtifacts selects the artifacts of the names, every one when there is
215// none. A name that no artifact has is an error.
216func pickArtifacts(all []*ermite.Artifact, names []string) ([]*ermite.Artifact, error) {
217 if len(names) == 0 {
218 return all, nil
219 }
220 var out []*ermite.Artifact
221 for _, n := range names {
222 i := slices.IndexFunc(all, func(a *ermite.Artifact) bool { return a.GetName() == n })
223 if i < 0 {
224 return nil, fmt.Errorf("the run has no artifact %q", clean(n))
225 }
226 if !slices.Contains(out, all[i]) {
227 out = append(out, all[i])
228 }
229 }
230 return out, nil
231}
232
233// checkArtifactName refuses a name that is not a plain file name: the server's
234// rules exclude separators, but the name comes from a job.
235func checkArtifactName(name string) error {
236 if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\\x00") || !filepath.IsLocal(name) {
237 return fmt.Errorf("artifact %q: the name is not usable as a file name", clean(name))
238 }
239 return nil
240}
241
242// downloadArtifact streams an artifact into a temporary file of root, then
243// extracts it (a zip) or renames it to its name. It returns the path written,
244// relative to root.
245func downloadArtifact(ctx context.Context, c *client.Client, repo string, a *ermite.Artifact, root *os.Root, several bool) (string, error) {
246 var rnd [8]byte
247 _, _ = rand.Read(rnd[:])
248 tmp := ".ermite-download-" + hex.EncodeToString(rnd[:])
249 f, err := root.OpenFile(tmp, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600)
250 if err != nil {
251 return "", err
252 }
253 defer func() {
254 _ = f.Close()
255 _ = root.Remove(tmp)
256 }()
257 got, err := streamArtifact(ctx, c, repo, a.GetId(), f)
258 if err != nil {
259 return "", err
260 }
261 if got.GetMediaType() != artifactZip {
262 if err := f.Close(); err != nil {
263 return "", err
264 }
265 if _, err := root.Lstat(a.GetName()); err == nil {
266 return "", fmt.Errorf("%s already exists", a.GetName())
267 } else if !errors.Is(err, fs.ErrNotExist) {
268 return "", err
269 }
270 return a.GetName(), root.Rename(tmp, a.GetName())
271 }
272 into, to := root, "."
273 if several {
274 to = a.GetName()
275 if err := root.MkdirAll(to, 0o755); err != nil {
276 return "", err
277 }
278 if into, err = root.OpenRoot(to); err != nil {
279 return "", err
280 }
281 defer func() { _ = into.Close() }()
282 }
283 size, err := f.Seek(0, io.SeekEnd)
284 if err != nil {
285 return "", err
286 }
287 if err := extractZip(f, size, into, maxExtractBytes); err != nil {
288 return "", err
289 }
290 return to, nil
291}
292
293// streamArtifact writes the content of an artifact to w and returns the
294// artifact, checking that the content has its size.
295func streamArtifact(ctx context.Context, c *client.Client, repo string, id int64, w io.Writer) (*ermite.Artifact, error) {
296 stream, err := c.Artifacts().DownloadArtifact(ctx, connect.NewRequest(&ermite.DownloadArtifactRequest{Repo: repo, ArtifactId: id}))
297 if err != nil {
298 return nil, err
299 }
300 defer func() { _ = stream.Close() }()
301 var a *ermite.Artifact
302 var n int64
303 for stream.Receive() {
304 msg := stream.Msg()
305 if a == nil {
306 if a = msg.GetArtifact(); a == nil {
307 return nil, errors.New("the server did not describe the artifact")
308 }
309 }
310 chunk := msg.GetChunk()
311 if n+int64(len(chunk)) > a.GetSize() {
312 return nil, errors.New("the server sent more than the size of the artifact")
313 }
314 if _, err := w.Write(chunk); err != nil {
315 return nil, err
316 }
317 n += int64(len(chunk))
318 }
319 if err := stream.Err(); err != nil {
320 return nil, err
321 }
322 if a == nil || n != a.GetSize() {
323 return nil, errors.New("the download ended before the end of the artifact")
324 }
325 return a, nil
326}
327
328// extractZip writes the files of the archive into root. Every entry must be
329// a local path (no absolute path, no "..") and a directory or a regular file:
330// a symbolic link or any other kind fails the whole extraction. Files that
331// exist are not overwritten, and the extraction writes at most limit bytes.
332// os.Root keeps every write inside the directory, whatever links it holds.
333func extractZip(r io.ReaderAt, size int64, root *os.Root, limit int64) error {
334 zr, err := zip.NewReader(r, size)
335 if err != nil && !errors.Is(err, zip.ErrInsecurePath) {
336 return fmt.Errorf("not a zip archive: %w", err)
337 }
338 if len(zr.File) > maxZipEntries {
339 return fmt.Errorf("the archive has more than %d entries", maxZipEntries)
340 }
341 var total uint64
342 for _, f := range zr.File {
343 if err := checkZipEntry(f); err != nil {
344 return err
345 }
346 total += f.UncompressedSize64
347 }
348 if limit < 0 || total > uint64(limit) {
349 return fmt.Errorf("the archive expands to more than %s", humanBytes(limit))
350 }
351 remaining := limit
352 for _, f := range zr.File {
353 name := path.Clean(f.Name)
354 if f.FileInfo().IsDir() {
355 if err := root.MkdirAll(filepath.FromSlash(name), 0o755); err != nil {
356 return err
357 }
358 continue
359 }
360 if dir := path.Dir(name); dir != "." {
361 if err := root.MkdirAll(filepath.FromSlash(dir), 0o755); err != nil {
362 return err
363 }
364 }
365 n, err := extractZipFile(f, root, filepath.FromSlash(name), remaining)
366 if err != nil {
367 return err
368 }
369 remaining -= n
370 }
371 return nil
372}
373
374// checkZipEntry refuses an entry that is not a local directory or regular file.
375func checkZipEntry(f *zip.File) error {
376 name := f.Name
377 if name == "" || strings.HasPrefix(name, "/") || strings.Contains(name, "\\") || strings.ContainsRune(name, 0) ||
378 slices.Contains(strings.Split(name, "/"), "..") || !filepath.IsLocal(filepath.FromSlash(path.Clean(name))) {
379 return fmt.Errorf("the archive has an unsafe path %q", clean(name))
380 }
381 mode := f.Mode()
382 if !mode.IsDir() && !mode.IsRegular() {
383 return fmt.Errorf("the archive has %q, which is not a regular file or a directory", clean(name))
384 }
385 return nil
386}
387
388// extractZipFile writes one file of the archive, at most limit bytes.
389func extractZipFile(f *zip.File, root *os.Root, name string, limit int64) (int64, error) {
390 perm := os.FileMode(0o644)
391 if f.Mode()&0o100 != 0 {
392 perm = 0o755
393 }
394 rc, err := f.Open()
395 if err != nil {
396 return 0, err
397 }
398 defer func() { _ = rc.Close() }()
399 out, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm)
400 if err != nil {
401 return 0, err
402 }
403 n, err := io.Copy(out, io.LimitReader(rc, limit+1))
404 if cerr := out.Close(); err == nil {
405 err = cerr
406 }
407 if err == nil && n > limit {
408 err = fmt.Errorf("the archive expands to more than declared")
409 }
410 return n, err
411}