1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3// Package githubapi serves the subset of GitHub's REST API that the actions
4// running on ermite call under github.api_url (/-/github/api, ADR 0025 CI-2):
5//
6// GET /repos/{owner}/{repo}
7// GET /repos/{owner}/{repo}/hash-algorithm
8// GET /repos/{owner}/{repo}/tarball[/{ref}]
9// GET /repos/{owner}/{repo}/pulls/{number}/files (dorny/paths-filter on pull_request)
10//
11// Everything else under the prefix is GitHub's 404 JSON, so that an action
12// calling an endpoint of a feature ermite lacks fails as it does on a GitHub
13// instance without it, never silently succeeds. The auth is the forge's: a
14// token (stored, or the repository token of a job) as the password of Basic with
15// any user, or as "Bearer" or "token" credentials; no credentials is anonymous,
16// which reads public repositories.
17package githubapi
18
19import (
20 "cmp"
21 "context"
22 "errors"
23 "io"
24 "net/http"
25 "net/netip"
26 "strconv"
27 "strings"
28 "sync"
29 "time"
30
31 "github.com/bnema/zerowrap"
32
33 "bnema.dev/ermite/internal/clientlimit"
34 "bnema.dev/ermite/internal/domain"
35 "bnema.dev/ermite/internal/ports"
36)
37
38// Prefix is where the API is mounted.
39const Prefix = "/-/github/api"
40
41// Patterns are the ServeMux patterns the handler serves: its whole prefix.
42var Patterns = []string{Prefix + "/"}
43
44// notFoundBody is GitHub's body for an unknown endpoint or repository.
45const notFoundBody = `{"message":"Not Found","documentation_url":"https://docs.github.com/rest"}`
46
47// retryAfter is the Retry-After, in seconds, of an answer that asks the client to
48// come back (429 and 503).
49const retryAfter = "5"
50
51// Bounds on the archives written at once.
52const (
53 // DefaultMaxArchives is the archives written at once on the server.
54 DefaultMaxArchives = 16
55 // DefaultMaxArchivesPerClient is the archives written at once for one client
56 // (see clientlimit.Key). A runner host runs several jobs at once, each
57 // downloading its actions, and all come from one address; eight is half of
58 // the server's slots, so one host cannot take them all. Beyond it the answer
59 // is 429 with Retry-After: the runner waits and tries again.
60 DefaultMaxArchivesPerClient = 8
61 // DefaultWriteIdleTimeout is how long a write of the archive may wait for the
62 // client. The deadline moves forward with every chunk: an archive is bounded
63 // by its size, not by its duration, but a client that stops reading is cut.
64 DefaultWriteIdleTimeout = 2 * time.Minute
65)
66
67// Config configures a Handler.
68type Config struct {
69 Auth ports.AuthService
70 API ports.GitHubAPIService
71 // Releases and ReleaseDownload enable the release REST routes. Nil services
72 // leave their routes unavailable with GitHub's 404 JSON.
73 Releases ports.GitHubReleaseAPI
74 ReleaseDownload ports.ReleaseService
75 // TrustedProxies are the networks allowed to set X-Forwarded-For.
76 TrustedProxies []netip.Prefix
77 // Limiter counts failed authentications; nil builds a default one.
78 Limiter ports.FailureLimiter
79 // MaxArchives bounds the archives written at once (each runs a git
80 // process); zero means DefaultMaxArchives. MaxArchivesPerClient bounds them
81 // per client; zero means DefaultMaxArchivesPerClient. WriteIdleTimeout is
82 // the idle write deadline; zero means DefaultWriteIdleTimeout.
83 MaxArchives int
84 MaxArchivesPerClient int
85 WriteIdleTimeout time.Duration
86}
87
88// Handler serves the REST subset.
89type Handler struct {
90 cfg Config
91 limiter ports.FailureLimiter
92 archives chan struct{}
93 perCli int
94 idle time.Duration
95
96 mu sync.Mutex // guards clients
97 clients map[string]int // archives being written, per clientlimit.Key
98}
99
100// New builds a Handler.
101func New(cfg Config) *Handler {
102 h := &Handler{cfg: cfg, limiter: cfg.Limiter}
103 if h.limiter == nil {
104 h.limiter = clientlimit.NewLimiter(nil)
105 }
106 n := cfg.MaxArchives
107 if n <= 0 {
108 n = DefaultMaxArchives
109 }
110 h.archives = make(chan struct{}, n)
111 h.perCli = cmp.Or(cfg.MaxArchivesPerClient, DefaultMaxArchivesPerClient)
112 h.idle = cmp.Or(cfg.WriteIdleTimeout, DefaultWriteIdleTimeout)
113 h.clients = make(map[string]int)
114 return h
115}
116
117// ServeHTTP implements http.Handler.
118func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
119 w.Header().Set("Cache-Control", "no-store")
120 w.Header().Set("X-Content-Type-Options", "nosniff")
121 rest := strings.TrimPrefix(r.URL.Path, Prefix)
122 kind, owner, repo, ref, ok := route(rest)
123 var number int64
124 if kind == "pullfiles" {
125 number, _ = strconv.ParseInt(ref, 10, 64)
126 }
127 if !ok {
128 notFound(w)
129 return
130 }
131 if strings.HasPrefix(kind, "release") && h.cfg.Releases == nil {
132 notFound(w)
133 return
134 }
135 allowed := releaseMethods(kind)
136 if !methodAllowed(r.Method, allowed) {
137 w.Header().Set("Allow", allowed)
138 jsonError(w, http.StatusMethodNotAllowed, "Method Not Allowed")
139 return
140 }
141
142 ctx := r.Context()
143 addr := clientlimit.ClientAddr(r, h.cfg.TrustedProxies)
144 if blocked, _ := h.limiter.Blocked(ctx, addr); blocked {
145 jsonError(w, http.StatusTooManyRequests, "API rate limit exceeded")
146 return
147 }
148 p, err := h.authenticate(r, addr)
149 if err != nil {
150 if errors.Is(err, domain.ErrUnauthenticated) {
151 jsonError(w, http.StatusUnauthorized, "Bad credentials")
152 return
153 }
154 log := zerowrap.FromCtx(ctx)
155 log.Error().Err(err).Msg("authenticate github api request")
156 jsonError(w, http.StatusInternalServerError, "Server Error")
157 return
158 }
159
160 if strings.HasPrefix(kind, "release") {
161 h.releaseRequest(w, r, p, kind, owner, repo, ref, clientlimit.Key(addr))
162 return
163 }
164 switch kind {
165 case "repo":
166 body, err := h.cfg.API.GetRepo(ctx, p, owner, repo)
167 if err != nil {
168 h.fail(ctx, w, err)
169 return
170 }
171 writeJSON(w, r, body)
172 case "hash":
173 alg, err := h.cfg.API.HashAlgorithm(ctx, p, owner, repo)
174 if err != nil {
175 h.fail(ctx, w, err)
176 return
177 }
178 writeJSON(w, r, []byte(`{"hash_algorithm":`+strconv.Quote(alg)+`}`))
179 case "tarball":
180 h.tarball(w, r, p, owner, repo, ref, clientlimit.Key(addr))
181 case "pullfiles":
182 h.pullFiles(w, r, p, owner, repo, number)
183 }
184}
185
186// route recognises the endpoints. ref is "" for /tarball.
187func route(rest string) (kind, owner, repo, ref string, ok bool) {
188 // go-github's enterprise client (used by GoReleaser) appends api/v3
189 // to a custom API base. Keep the canonical URLs, accepting that alias too.
190 if strings.HasPrefix(rest, "/api/v3/") {
191 rest = strings.TrimPrefix(rest, "/api/v3")
192 }
193 if kind, owner, repo, ref, ok = releaseRoute(rest); ok {
194 return
195 }
196 parts := strings.Split(strings.TrimPrefix(rest, "/"), "/")
197 if len(parts) < 3 || parts[0] != "repos" || parts[1] == "" || parts[2] == "" {
198 return "", "", "", "", false
199 }
200 owner, repo = parts[1], parts[2]
201 switch {
202 case len(parts) == 3:
203 return "repo", owner, repo, "", true
204 case len(parts) == 4 && parts[3] == "hash-algorithm":
205 return "hash", owner, repo, "", true
206 case len(parts) == 6 && parts[3] == "pulls" && parts[5] == "files" && isNumber(parts[4]):
207 // ref carries the number of the pull request.
208 return "pullfiles", owner, repo, parts[4], true
209 case len(parts) >= 4 && parts[3] == "tarball":
210 // The ref may hold slashes (feature/x): everything after tarball/.
211 return "tarball", owner, repo, strings.Join(parts[4:], "/"), true
212 }
213 return "", "", "", "", false
214}
215
216// isNumber reports whether s is a positive decimal number.
217func isNumber(s string) bool {
218 n, err := strconv.ParseInt(s, 10, 64)
219 return err == nil && n > 0 && strconv.FormatInt(n, 10) == s
220}
221
222// pullFiles answers the files of a pull request, a page at a time as GitHub
223// does: page and per_page in the query (30 and 100 at most), the other pages
224// in the Link header. A parameter that is not a positive number is the default,
225// as GitHub treats it.
226func (h *Handler) pullFiles(w http.ResponseWriter, r *http.Request, p domain.Principal, owner, repo string, number int64) {
227 q := r.URL.Query()
228 page, _ := strconv.Atoi(q.Get("page"))
229 perPage, _ := strconv.Atoi(q.Get("per_page"))
230 body, link, err := h.cfg.API.ListPullFiles(r.Context(), p, owner, repo, number, page, perPage)
231 if err != nil {
232 if errors.Is(err, domain.ErrForbidden) {
233 jsonError(w, http.StatusForbidden, "Resource not accessible by integration")
234 return
235 }
236 h.fail(r.Context(), w, err)
237 return
238 }
239 if link != "" {
240 w.Header().Set("Link", link)
241 }
242 writeJSON(w, r, body)
243}
244
245// tarball answers an archive request. A HEAD resolves the ref and sends the
246// headers of the archive, and runs no git. A GET streams the archive; it is not
247// a 302 to a codeload: ermite serves the bytes itself, which the runner follows
248// without a second credential.
249func (h *Handler) tarball(w http.ResponseWriter, r *http.Request, p domain.Principal, owner, repo, ref, client string) {
250 ctx := r.Context()
251 if r.Method == http.MethodHead {
252 name, err := h.cfg.API.TarballName(ctx, p, owner, repo, ref)
253 if err != nil {
254 h.fail(ctx, w, err)
255 return
256 }
257 setArchiveHeaders(w, name)
258 w.WriteHeader(http.StatusOK)
259 return
260 }
261 release, ok := h.acquire(w, client)
262 if !ok {
263 return
264 }
265 defer release()
266 rc := http.NewResponseController(w)
267 started := false
268 err := h.cfg.API.Tarball(ctx, p, owner, repo, ref, func(name string) io.Writer {
269 started = true
270 setArchiveHeaders(w, name)
271 return &idleWriter{w: w, rc: rc, idle: h.idle}
272 })
273 switch {
274 case err == nil:
275 case started:
276 // The status is out: the client sees a truncated archive. Log it.
277 log := zerowrap.FromCtx(ctx)
278 log.Warn().Err(err).Msg("github api archive cut short")
279 default:
280 h.fail(ctx, w, err)
281 }
282}
283
284// acquire takes an archive slot for the client, or answers 429 (the client
285// already has its share) or 503 (the server writes all it can) and reports
286// false. It never waits: an archive can take minutes.
287func (h *Handler) acquire(w http.ResponseWriter, client string) (release func(), ok bool) {
288 h.mu.Lock()
289 if h.clients[client] >= h.perCli {
290 h.mu.Unlock()
291 w.Header().Set("Retry-After", retryAfter)
292 jsonError(w, http.StatusTooManyRequests, "Too many archives in progress")
293 return nil, false
294 }
295 h.clients[client]++
296 h.mu.Unlock()
297 releaseClient := func() {
298 h.mu.Lock()
299 if h.clients[client]--; h.clients[client] <= 0 {
300 delete(h.clients, client)
301 }
302 h.mu.Unlock()
303 }
304 select {
305 case h.archives <- struct{}{}:
306 return func() { <-h.archives; releaseClient() }, true
307 default:
308 releaseClient()
309 w.Header().Set("Retry-After", retryAfter)
310 jsonError(w, http.StatusServiceUnavailable, "Server busy")
311 return nil, false
312 }
313}
314
315// idleWriter moves the write deadline of the connection forward before each
316// chunk, so that only a client that stops reading is cut. Where the response
317// writer cannot set a deadline, the writes are not bounded here.
318type idleWriter struct {
319 w io.Writer
320 rc *http.ResponseController
321 idle time.Duration
322}
323
324func (i *idleWriter) Write(b []byte) (int, error) {
325 _ = i.rc.SetWriteDeadline(time.Now().Add(i.idle))
326 return i.w.Write(b)
327}
328
329func setArchiveHeaders(w http.ResponseWriter, name string) {
330 w.Header().Set("Content-Type", "application/x-gzip")
331 w.Header().Set("Content-Disposition", "attachment; filename="+name)
332}
333
334// authenticate reads the credentials of the request. None is the anonymous
335// principal; credentials that are not valid count as a failure of the client
336// address.
337func (h *Handler) authenticate(r *http.Request, addr string) (domain.Principal, error) {
338 secret, present := credentials(r)
339 if !present {
340 return domain.Anonymous(), nil
341 }
342 p, err := h.cfg.Auth.Authenticate(r.Context(), secret)
343 if err != nil {
344 if errors.Is(err, domain.ErrUnauthenticated) {
345 h.limiter.Fail(r.Context(), addr)
346 }
347 return domain.Anonymous(), err
348 }
349 return p, nil
350}
351
352// credentials extracts the token of an Authorization header: the password of
353// Basic (the user is ignored, checkout sends x-access-token), or the value of
354// "Bearer" or "token".
355func credentials(r *http.Request) (secret string, present bool) {
356 auth := r.Header.Get("Authorization")
357 if auth == "" {
358 return "", false
359 }
360 if _, pass, ok := r.BasicAuth(); ok {
361 return pass, true
362 }
363 if scheme, tok, ok := strings.Cut(auth, " "); ok && (strings.EqualFold(scheme, "Bearer") || strings.EqualFold(scheme, "token")) {
364 return strings.TrimSpace(tok), true
365 }
366 return "", true
367}
368
369// fail answers an error of a use case. A repository the caller cannot read is
370// as not found as one that does not exist.
371func (h *Handler) fail(ctx context.Context, w http.ResponseWriter, err error) {
372 switch {
373 case errors.Is(err, domain.ErrNotFound), domain.IsInvalid(err):
374 notFound(w)
375 case errors.Is(err, domain.ErrResourceExhausted):
376 w.Header().Set("Retry-After", retryAfter)
377 jsonError(w, http.StatusServiceUnavailable, "Server busy")
378 case errors.Is(err, context.Canceled):
379 default:
380 log := zerowrap.FromCtx(ctx)
381 log.Error().Err(err).Msg("github api request failed")
382 jsonError(w, http.StatusInternalServerError, "Server Error")
383 }
384}
385
386func notFound(w http.ResponseWriter) {
387 w.Header().Set("Content-Type", "application/json; charset=utf-8")
388 w.WriteHeader(http.StatusNotFound)
389 _, _ = io.WriteString(w, notFoundBody)
390}
391
392func jsonError(w http.ResponseWriter, status int, msg string) {
393 w.Header().Set("Content-Type", "application/json; charset=utf-8")
394 w.WriteHeader(status)
395 _, _ = io.WriteString(w, `{"message":`+strconv.Quote(msg)+`,"documentation_url":"https://docs.github.com/rest"}`)
396}
397
398func writeJSON(w http.ResponseWriter, r *http.Request, body []byte) {
399 w.Header().Set("Content-Type", "application/json; charset=utf-8")
400 w.Header().Set("Content-Length", strconv.Itoa(len(body)))
401 w.WriteHeader(http.StatusOK)
402 if r.Method != http.MethodHead {
403 _, _ = w.Write(body)
404 }
405}