1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package usecase
4
5import (
6 "bytes"
7 "context"
8 "crypto/sha256"
9 "encoding/base64"
10 "encoding/binary"
11 "encoding/json"
12 "net/url"
13 "strconv"
14 "strings"
15 "time"
16
17 "github.com/bnema/zerowrap"
18
19 "bnema.dev/ermite/internal/domain"
20)
21
22// This file builds the "github" context of a push and the "job" keys the
23// server sends, to match what github.com gives (recordings of
24// ermite-conformance, normalised by its normalize.go).
25//
26// Value provenance, used in the comments below:
27// - real: a value ermite has (repository name, visibility, description,
28// dates, commits, default branch, configuration);
29// - derived: computed from real values by a stable rule (ids, node ids,
30// URLs);
31// - constant: GitHub's value for a repository or account with no such
32// feature, or a placeholder where ermite has no concept (no avatars,
33// stars, forks, issues...).
34
35const (
36 // ghSecretSource is github.secret_source: where the secrets of the job
37 // come from. Constant: ermite has one source.
38 ghSecretSource = "Actions"
39 // ghCacheLimit is github.artifact_cache_size_limit, in GB, as recorded on
40 // github.com. Constant.
41 ghCacheLimit = "10"
42 // apiPath is where the GitHub-shaped API base of the payload lives under
43 // the public URL. github.api_url is that base, github.graphql_url is
44 // <base>/graphql and the *_url keys of the repository and the users are
45 // <base>/repos/... and <base>/users/..., as on github.com. ermite serves
46 // none of them yet: they are identifiers in the shape of GitHub's.
47 apiPath = "/-/github/api"
48)
49
50// pair is a key and a value of an ordered JSON object.
51type pair struct {
52 k string
53 v any
54}
55
56// ordered is a JSON object that keeps the order of its keys, as the github
57// context of GitHub does (toJSON(github) prints that order).
58type ordered []pair
59
60func (o ordered) MarshalJSON() ([]byte, error) {
61 var b bytes.Buffer
62 b.WriteByte('{')
63 for i, p := range o {
64 if i > 0 {
65 b.WriteByte(',')
66 }
67 k, err := json.Marshal(p.k)
68 if err != nil {
69 return nil, err
70 }
71 v, err := json.Marshal(p.v)
72 if err != nil {
73 return nil, err
74 }
75 b.Write(k)
76 b.WriteByte(':')
77 b.Write(v)
78 }
79 b.WriteByte('}')
80 return b.Bytes(), nil
81}
82
83// stableID is a positive number that depends only on the kind and the name:
84// the id of a user or of a repository, which ermite does not number. It fits
85// in a JavaScript number (GitHub's own ids are well below 2^53).
86func stableID(kind, name string) int64 {
87 sum := sha256.Sum256([]byte(kind + "\x00" + name))
88 return int64(binary.BigEndian.Uint64(sum[:8])>>20) + 1<<20
89}
90
91// nodeID is the legacy form of a GraphQL node id: base64 of "<n>:<Type><id>"
92// (GitHub: "04:User5212137" is "MDQ6VXNlcjUyMTIxMzc=").
93func nodeID(prefix, typ string, id int64) string {
94 return base64.StdEncoding.EncodeToString([]byte(prefix + ":" + typ + strconv.FormatInt(id, 10)))
95}
96
97// ghIdentity is the account a push is attributed to.
98type ghIdentity struct {
99 login string
100 id int64
101}
102
103// identity maps the actor of a run to a login: "owner" and "root" (the
104// owner's session or the root token) are the owner of the forge, a token
105// label is itself.
106func (c *CI) identity(actor string) ghIdentity {
107 login := actor
108 if actor == "owner" || actor == "root" || actor == "" {
109 login = c.cfg.Owner
110 }
111 return ghIdentity{login: login, id: stableID("user", login)}
112}
113
114// ghURLs holds the URL bases of the payload of one repository.
115type ghURLs struct {
116 server string // github.server_url
117 api string // github.api_url
118 host string // host (and port) of the server, for git:// and ssh URLs
119 owner string
120 name string
121 full string // <owner>/<repo>
122 repo string // <server>/<owner>/<repo>, the page GitHub's html_url points at
123}
124
125func (c *CI) urls(repo domain.Repo) ghURLs {
126 u := ghURLs{server: c.cfg.PublicURL, owner: c.cfg.Owner, name: repo.Name.String()}
127 if p, err := url.Parse(c.cfg.PublicURL); err == nil {
128 u.host = p.Host
129 }
130 u.api = u.server + apiPath
131 u.full = u.owner + "/" + u.name
132 u.repo = u.server + "/" + u.full
133 return u
134}
135
136// userObj is the account object of an event (sender, repository owner).
137// Derived: ids from the login; URLs from the API base. Constant: type, no
138// avatar (ermite has none: "" is what GitHub gives for gravatar_id too), not
139// a site admin.
140func (u ghURLs) userObj(id ghIdentity, owner bool) map[string]any {
141 api := u.api + "/users/" + id.login
142 o := map[string]any{
143 "avatar_url": "", "events_url": api + "/events{/privacy}", "followers_url": api + "/followers",
144 "following_url": api + "/following{/other_user}", "gists_url": api + "/gists{/gist_id}", "gravatar_id": "",
145 "html_url": u.server + "/" + id.login, "id": id.id, "login": id.login,
146 "node_id": nodeID("04", "User", id.id),
147 "organizations_url": api + "/orgs", "received_events_url": api + "/received_events", "repos_url": api + "/repos",
148 "site_admin": false, "starred_url": api + "/starred{/owner}{/repo}", "subscriptions_url": api + "/subscriptions",
149 "type": "User", "url": api, "user_view_type": "public",
150 }
151 if owner {
152 o["email"], o["name"] = "", id.login
153 }
154 return o
155}
156
157// repoLinks are the URL templates of the repository object, relative to
158// <api>/repos/<owner>/<repo>. Derived (constant templates, as GitHub's).
159var repoLinks = map[string]string{
160 "archive_url": "/{archive_format}{/ref}", "assignees_url": "/assignees{/user}", "blobs_url": "/git/blobs{/sha}",
161 "branches_url": "/branches{/branch}", "collaborators_url": "/collaborators{/collaborator}",
162 "comments_url": "/comments{/number}", "commits_url": "/commits{/sha}", "compare_url": "/compare/{base}...{head}",
163 "contents_url": "/contents/{+path}", "contributors_url": "/contributors", "deployments_url": "/deployments",
164 "downloads_url": "/downloads", "events_url": "/events", "forks_url": "/forks", "git_commits_url": "/git/commits{/sha}",
165 "git_refs_url": "/git/refs{/sha}", "git_tags_url": "/git/tags{/sha}", "hooks_url": "/hooks",
166 "issue_comment_url": "/issues/comments{/number}", "issue_events_url": "/issues/events{/number}",
167 "issues_url": "/issues{/number}", "keys_url": "/keys{/key_id}", "labels_url": "/labels{/name}",
168 "languages_url": "/languages", "merges_url": "/merges", "milestones_url": "/milestones{/number}",
169 "notifications_url": "/notifications{?since,all,participating}", "pulls_url": "/pulls{/number}",
170 "releases_url": "/releases{/id}", "stargazers_url": "/stargazers", "statuses_url": "/statuses/{sha}",
171 "subscribers_url": "/subscribers", "subscription_url": "/subscription", "tags_url": "/tags", "teams_url": "/teams",
172 "trees_url": "/git/trees{/sha}", "url": "",
173}
174
175// repoObj is event.repository: the 83 keys of GitHub's push payload.
176//
177// real: name, full_name, private, visibility, description, default_branch,
178// master_branch, created_at, pushed_at, updated_at
179// derived: id, node_id, owner, html_url, svn_url, clone_url, git_url,
180// ssh_url, url and the other *_url keys
181// constant: the rest (no forks, stars, issues, pages..., the features of a
182// new GitHub repository; license, language, homepage and
183// mirror_url null; size 0)
184func (c *CI) repoObj(repo domain.Repo, u ghURLs, defaultBranch string, at time.Time) map[string]any {
185 owner := c.identity("owner")
186 id := stableID("repo", u.full)
187 pushed := repo.PushedAt
188 if pushed.IsZero() {
189 pushed = at
190 }
191 var desc any
192 if repo.Description != "" {
193 desc = repo.Description
194 }
195 r := map[string]any{
196 "allow_forking": true, "archived": false, "clone_url": u.repo + ".git",
197 "created_at": repo.CreatedAt.Unix(), "default_branch": defaultBranch, "description": desc,
198 "disabled": false, "fork": false, "forks": 0, "forks_count": 0, "full_name": u.full,
199 "git_url": "git://" + u.host + "/" + u.full + ".git",
200 "has_discussions": false, "has_downloads": true, "has_issues": true, "has_pages": false, "has_projects": true,
201 "has_pull_requests": true, "has_wiki": true, "homepage": nil, "html_url": u.repo, "id": id,
202 "is_template": false, "language": nil, "license": nil, "master_branch": defaultBranch, "mirror_url": nil,
203 "name": u.name, "node_id": nodeID("010", "Repository", id), "open_issues": 0, "open_issues_count": 0,
204 "owner": u.userObj(owner, true),
205 "private": repo.Visibility == domain.VisibilityPrivate, "pull_request_creation_policy": "all",
206 "pushed_at": pushed.Unix(), "size": 0, "ssh_url": "git@" + u.host + ":" + u.full + ".git",
207 "stargazers": 0, "stargazers_count": 0, "svn_url": u.repo, "topics": []string{},
208 "updated_at": pushed.UTC().Format(time.RFC3339), "visibility": string(repo.Visibility),
209 "watchers": 0, "watchers_count": 0, "web_commit_signoff_required": false,
210 }
211 for k, suffix := range repoLinks {
212 r[k] = u.api + "/repos/" + u.full + suffix
213 }
214 return r
215}
216
217// commitJSON is one commit of the push event: GitHub's push payload has
218// exactly these keys in the recordings (no added/removed/modified, no
219// username).
220func (c *CI) commitJSON(ctx context.Context, repo domain.Repo, u ghURLs, cm domain.Commit) map[string]any {
221 tree := ""
222 if id, err := c.cfg.Git.CommitTree(ctx, repo.Name, cm.ID); err == nil {
223 tree = id.String()
224 }
225 return map[string]any{
226 "id": cm.ID.String(), "tree_id": tree, "distinct": true, "message": cm.Message,
227 "timestamp": cm.Committer.Time.UTC().Format(time.RFC3339),
228 "url": u.repo + "/commit/" + cm.ID.String(),
229 "author": person(cm.Author.Name, cm.Author.Email), "committer": person(cm.Committer.Name, cm.Committer.Email),
230 }
231}
232
233// pushPayload is the part of the push event that is the same for every
234// workflow of one ref: the commits, each with its tree id (one Git call per
235// commit). It is built once per ref and shared by the probes and the runs.
236type pushPayload struct {
237 commits []any
238 head map[string]any
239}
240
241// pushPayload builds the commit part of the push event of f.
242func (c *CI) pushPayload(ctx context.Context, repo domain.Repo, f pushFacts) *pushPayload {
243 u := c.urls(repo)
244 p := &pushPayload{commits: make([]any, 0, len(f.commits)), head: c.commitJSON(ctx, repo, u, f.head)}
245 for _, cm := range f.commits {
246 p.commits = append(p.commits, c.commitJSON(ctx, repo, u, cm))
247 }
248 return p
249}
250
251func person(name, email string) map[string]any { return map[string]any{"name": name, "email": email} }
252
253// githubJSON builds the "github" context of the run as GitHub does for a push
254// (the job-dependent keys, job and run_attempt, are filled in when a job is
255// planned). Keys the runner owns (token, workspace, action, event_path, path,
256// env, output, state, step_summary, artifacts...) are not here.
257//
258// The order of the keys is GitHub's, so that toJSON(github) prints the same.
259// Ids and counters are strings, so is retention_days; "compare" has
260// 12-character ids: the commit page for a created ref with one commit,
261// "<first>^...<after>" for a created ref with several, "<before>...<after>" for
262// an update.
263//
264// The commits of the event come from f.payload, built once per ref; without
265// one they are read from Git here.
266func (c *CI) githubJSON(ctx context.Context, repo domain.Repo, run domain.CIRun, f pushFacts) string {
267 u := c.urls(repo)
268 id := c.identity(run.Actor)
269 owner := c.identity("owner")
270 refName, refType := strings.TrimPrefix(run.Ref, "refs/heads/"), "branch"
271 if f.isTag {
272 refName, refType = strings.TrimPrefix(run.Ref, "refs/tags/"), "tag"
273 }
274 before := f.update.Old
275 if f.created {
276 before = domain.ZeroOID
277 }
278 compare := u.repo + "/compare/" + shortID(before) + "..." + shortID(f.update.New)
279 switch {
280 case f.created && len(f.commits) > 1:
281 compare = u.repo + "/compare/" + shortID(f.commits[0].ID.String()) + "^..." + shortID(f.update.New)
282 case f.created:
283 compare = u.repo + "/commit/" + shortID(f.update.New)
284 }
285 payload := f.payload
286 if payload == nil {
287 payload = c.pushPayload(ctx, repo, f)
288 }
289 event := map[string]any{
290 "ref": run.Ref, "before": before, "after": f.update.New, "created": f.created, "deleted": false, "forced": f.forced,
291 "base_ref": nil, "compare": compare, "commits": payload.commits, "head_commit": payload.head,
292 "pusher": map[string]any{"name": id.login, "email": ""},
293 "repository": c.repoObj(repo, u, f.defaultBranch, run.CreatedAt),
294 "sender": u.userObj(id, false),
295 }
296 repoID := strconv.FormatInt(stableID("repo", u.full), 10)
297 gh := ordered{
298 {"job", ""}, {"ref", run.Ref}, {"sha", run.Sha}, {"repository", u.full}, {"repository_owner", c.cfg.Owner},
299 {"repository_owner_id", strconv.FormatInt(owner.id, 10)}, {"repositoryUrl", "git://" + u.host + "/" + u.full + ".git"},
300 {"run_id", strconv.FormatInt(run.ID, 10)}, {"run_number", strconv.FormatInt(run.Number, 10)},
301 {"retention_days", strconv.Itoa(c.retentionDays())}, {"run_attempt", "1"},
302 {"artifact_cache_size_limit", ghCacheLimit}, {"repository_visibility", string(repo.Visibility)},
303 {"actor_id", strconv.FormatInt(id.id, 10)}, {"actor", id.login}, {"workflow", run.WorkflowName},
304 {"head_ref", ""}, {"base_ref", ""}, {"event_name", "push"}, {"server_url", u.server},
305 {"api_url", u.api}, {"graphql_url", u.api + "/graphql"}, {"ref_name", refName}, {"ref_protected", false},
306 {"ref_type", refType}, {"secret_source", ghSecretSource}, {"event", event},
307 {"workflow_ref", u.full + "/" + run.WorkflowPath + "@" + run.Ref}, {"workflow_sha", run.Sha},
308 {"repository_id", repoID}, {"triggering_actor", id.login},
309 }
310 b, err := json.Marshal(gh)
311 if err != nil {
312 return "{}"
313 }
314 return string(b)
315}
316
317// Layout of job.check_run_id, 53 bits so that it is exact as a JavaScript
318// number (2^53 is the first integer that is not):
319//
320// bits 0-8 combo: matrix leg, 9 bits (0-511); a matrix has at most 256
321// bits 9-18 index: position of the job in the workflow file, 10 bits
322// (0-1023); ermite has no limit of its own on the number
323// of jobs
324// bits 19-25 attempt: run attempt, 7 bits (0-127), so that every re-run
325// gets new ids, as on GitHub
326// bits 26-52 run: run id, 27 bits (below 134 million)
327//
328// A field that overflows would wrap into another instance's id, so the whole
329// id is 0 then (see checkRunID).
330const (
331 checkComboBits = 9
332 checkIndexBits = 10
333 checkAttemptBits = 7
334 checkRunBits = 53 - checkComboBits - checkIndexBits - checkAttemptBits
335)
336
337// checkRunID is the number github.com shows in job.check_run_id: derived from
338// the run, its attempt and the position of the job instance, as ermite's
339// checks are not numbered. It is unique and stable per attempt, job and matrix
340// leg, and fits in a JavaScript number.
341//
342// The key is informational, so it never fails a job: a value outside its field
343// (which would wrap and give two instances the same id) is logged at warn with
344// the run, attempt and job, and the id falls back to 0.
345func checkRunID(ctx context.Context, run domain.CIRun, index, combo int) int64 {
346 attempt := int64(run.Attempt)
347 switch {
348 case run.ID < 0 || run.ID >= 1<<checkRunBits,
349 attempt < 0 || attempt >= 1<<checkAttemptBits,
350 index < 0 || index >= 1<<checkIndexBits,
351 combo < 0 || combo >= 1<<checkComboBits:
352 log := zerowrap.FromCtx(ctx)
353 log.Warn().Int64("run_id", run.ID).Int32("attempt", run.Attempt).Int("job_index", index).Int("leg", combo).
354 Msg("ci check_run_id is out of range: the job context shows 0")
355 return 0
356 }
357 return run.ID<<(checkAttemptBits+checkIndexBits+checkComboBits) |
358 attempt<<(checkIndexBits+checkComboBits) |
359 int64(index)<<checkComboBits |
360 int64(combo)
361}
362
363// jobKeys are the keys of the "job" context that the server knows, in
364// GitHub's order (the runner adds "status"). The runner copies the "job" key
365// of the contexts it is given into the job context.
366func (c *CI) jobKeys(ctx context.Context, run domain.CIRun, index, combo int) ordered {
367 id := checkRunID(ctx, run, index, combo)
368 u := c.urls(domain.Repo{Name: run.Repo})
369 return ordered{
370 {"check_run_id", id},
371 {"workflow_ref", u.full + "/" + run.WorkflowPath + "@" + run.Ref},
372 {"workflow_sha", run.Sha}, {"workflow_repository", u.full}, {"workflow_file_path", run.WorkflowPath},
373 }
374}