1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3// Package githubapi serves the subset of GitHub's REST API that the actions
4// running on ermite call under github.api_url (/-/github/api, ADR 0025 CI-2):
5//
6// GET /repos/{owner}/{repo}
7// GET /repos/{owner}/{repo}/hash-algorithm
8// GET /repos/{owner}/{repo}/tarball[/{ref}]
9// GET /repos/{owner}/{repo}/pulls/{number}/files (dorny/paths-filter on pull_request)
10//
11// Everything else under the prefix is GitHub's 404 JSON, so that an action
12// calling an endpoint of a feature ermite lacks fails as it does on a GitHub
13// instance without it, never silently succeeds. The auth is the forge's: a
14// token (stored, or the repository token of a job) as the password of Basic with
15// any user, or as "Bearer" or "token" credentials; no credentials is anonymous,
16// which reads public repositories.
17package githubapi
18
19import (
20 "cmp"
21 "context"
22 "errors"
23 "io"
24 "net/http"
25 "net/netip"
26 "strconv"
27 "strings"
28 "sync"
29 "time"
30
31 "github.com/bnema/zerowrap"
32
33 "bnema.dev/ermite/internal/clientlimit"
34 "bnema.dev/ermite/internal/domain"
35 "bnema.dev/ermite/internal/ports"
36)
37
38// Prefix is where the API is mounted.
39const Prefix = "/-/github/api"
40
41// Patterns are the ServeMux patterns the handler serves: its whole prefix.
42var Patterns = []string{Prefix + "/"}
43
44// notFoundBody is GitHub's body for an unknown endpoint or repository.
45const notFoundBody = `{"message":"Not Found","documentation_url":"https://docs.github.com/rest"}`
46
47// retryAfter is the Retry-After, in seconds, of an answer that asks the client to
48// come back (429 and 503).
49const retryAfter = "5"
50
51// Bounds on the archives written at once.
52const (
53 // DefaultMaxArchives is the archives written at once on the server.
54 DefaultMaxArchives = 16
55 // DefaultMaxArchivesPerClient is the archives written at once for one client
56 // (see clientlimit.Key). A runner host runs several jobs at once, each
57 // downloading its actions, and all come from one address; eight is half of
58 // the server's slots, so one host cannot take them all. Beyond it the answer
59 // is 429 with Retry-After: the runner waits and tries again.
60 DefaultMaxArchivesPerClient = 8
61 // DefaultWriteIdleTimeout is how long a write of the archive may wait for the
62 // client. The deadline moves forward with every chunk: an archive is bounded
63 // by its size, not by its duration, but a client that stops reading is cut.
64 DefaultWriteIdleTimeout = 2 * time.Minute
65)
66
67// Config configures a Handler.
68type Config struct {
69 Auth ports.AuthService
70 API ports.GitHubAPIService
71 // TrustedProxies are the networks allowed to set X-Forwarded-For.
72 TrustedProxies []netip.Prefix
73 // Limiter counts failed authentications; nil builds a default one.
74 Limiter ports.FailureLimiter
75 // MaxArchives bounds the archives written at once (each runs a git
76 // process); zero means DefaultMaxArchives. MaxArchivesPerClient bounds them
77 // per client; zero means DefaultMaxArchivesPerClient. WriteIdleTimeout is
78 // the idle write deadline; zero means DefaultWriteIdleTimeout.
79 MaxArchives int
80 MaxArchivesPerClient int
81 WriteIdleTimeout time.Duration
82}
83
84// Handler serves the REST subset.
85type Handler struct {
86 cfg Config
87 limiter ports.FailureLimiter
88 archives chan struct{}
89 perCli int
90 idle time.Duration
91
92 mu sync.Mutex // guards clients
93 clients map[string]int // archives being written, per clientlimit.Key
94}
95
96// New builds a Handler.
97func New(cfg Config) *Handler {
98 h := &Handler{cfg: cfg, limiter: cfg.Limiter}
99 if h.limiter == nil {
100 h.limiter = clientlimit.NewLimiter(nil)
101 }
102 n := cfg.MaxArchives
103 if n <= 0 {
104 n = DefaultMaxArchives
105 }
106 h.archives = make(chan struct{}, n)
107 h.perCli = cmp.Or(cfg.MaxArchivesPerClient, DefaultMaxArchivesPerClient)
108 h.idle = cmp.Or(cfg.WriteIdleTimeout, DefaultWriteIdleTimeout)
109 h.clients = make(map[string]int)
110 return h
111}
112
113// ServeHTTP implements http.Handler.
114func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
115 w.Header().Set("Cache-Control", "no-store")
116 w.Header().Set("X-Content-Type-Options", "nosniff")
117 rest := strings.TrimPrefix(r.URL.Path, Prefix)
118 kind, owner, repo, ref, ok := route(rest)
119 var number int64
120 if kind == "pullfiles" {
121 number, _ = strconv.ParseInt(ref, 10, 64)
122 }
123 if !ok {
124 notFound(w)
125 return
126 }
127 if r.Method != http.MethodGet && r.Method != http.MethodHead {
128 w.Header().Set("Allow", "GET, HEAD")
129 jsonError(w, http.StatusMethodNotAllowed, "Method Not Allowed")
130 return
131 }
132
133 ctx := r.Context()
134 addr := clientlimit.ClientAddr(r, h.cfg.TrustedProxies)
135 if blocked, _ := h.limiter.Blocked(ctx, addr); blocked {
136 jsonError(w, http.StatusTooManyRequests, "API rate limit exceeded")
137 return
138 }
139 p, err := h.authenticate(r, addr)
140 if err != nil {
141 if errors.Is(err, domain.ErrUnauthenticated) {
142 jsonError(w, http.StatusUnauthorized, "Bad credentials")
143 return
144 }
145 log := zerowrap.FromCtx(ctx)
146 log.Error().Err(err).Msg("authenticate github api request")
147 jsonError(w, http.StatusInternalServerError, "Server Error")
148 return
149 }
150
151 switch kind {
152 case "repo":
153 body, err := h.cfg.API.GetRepo(ctx, p, owner, repo)
154 if err != nil {
155 h.fail(ctx, w, err)
156 return
157 }
158 writeJSON(w, r, body)
159 case "hash":
160 alg, err := h.cfg.API.HashAlgorithm(ctx, p, owner, repo)
161 if err != nil {
162 h.fail(ctx, w, err)
163 return
164 }
165 writeJSON(w, r, []byte(`{"hash_algorithm":`+strconv.Quote(alg)+`}`))
166 case "tarball":
167 h.tarball(w, r, p, owner, repo, ref, clientlimit.Key(addr))
168 case "pullfiles":
169 h.pullFiles(w, r, p, owner, repo, number)
170 }
171}
172
173// route recognises the endpoints. ref is "" for /tarball.
174func route(rest string) (kind, owner, repo, ref string, ok bool) {
175 parts := strings.Split(strings.TrimPrefix(rest, "/"), "/")
176 if len(parts) < 3 || parts[0] != "repos" || parts[1] == "" || parts[2] == "" {
177 return "", "", "", "", false
178 }
179 owner, repo = parts[1], parts[2]
180 switch {
181 case len(parts) == 3:
182 return "repo", owner, repo, "", true
183 case len(parts) == 4 && parts[3] == "hash-algorithm":
184 return "hash", owner, repo, "", true
185 case len(parts) == 6 && parts[3] == "pulls" && parts[5] == "files" && isNumber(parts[4]):
186 // ref carries the number of the pull request.
187 return "pullfiles", owner, repo, parts[4], true
188 case len(parts) >= 4 && parts[3] == "tarball":
189 // The ref may hold slashes (feature/x): everything after tarball/.
190 return "tarball", owner, repo, strings.Join(parts[4:], "/"), true
191 }
192 return "", "", "", "", false
193}
194
195// isNumber reports whether s is a positive decimal number.
196func isNumber(s string) bool {
197 n, err := strconv.ParseInt(s, 10, 64)
198 return err == nil && n > 0 && strconv.FormatInt(n, 10) == s
199}
200
201// pullFiles answers the files of a pull request, a page at a time as GitHub
202// does: page and per_page in the query (30 and 100 at most), the other pages
203// in the Link header. A parameter that is not a positive number is the default,
204// as GitHub treats it.
205func (h *Handler) pullFiles(w http.ResponseWriter, r *http.Request, p domain.Principal, owner, repo string, number int64) {
206 q := r.URL.Query()
207 page, _ := strconv.Atoi(q.Get("page"))
208 perPage, _ := strconv.Atoi(q.Get("per_page"))
209 body, link, err := h.cfg.API.ListPullFiles(r.Context(), p, owner, repo, number, page, perPage)
210 if err != nil {
211 if errors.Is(err, domain.ErrForbidden) {
212 jsonError(w, http.StatusForbidden, "Resource not accessible by integration")
213 return
214 }
215 h.fail(r.Context(), w, err)
216 return
217 }
218 if link != "" {
219 w.Header().Set("Link", link)
220 }
221 writeJSON(w, r, body)
222}
223
224// tarball answers an archive request. A HEAD resolves the ref and sends the
225// headers of the archive, and runs no git. A GET streams the archive; it is not
226// a 302 to a codeload: ermite serves the bytes itself, which the runner follows
227// without a second credential.
228func (h *Handler) tarball(w http.ResponseWriter, r *http.Request, p domain.Principal, owner, repo, ref, client string) {
229 ctx := r.Context()
230 if r.Method == http.MethodHead {
231 name, err := h.cfg.API.TarballName(ctx, p, owner, repo, ref)
232 if err != nil {
233 h.fail(ctx, w, err)
234 return
235 }
236 setArchiveHeaders(w, name)
237 w.WriteHeader(http.StatusOK)
238 return
239 }
240 release, ok := h.acquire(w, client)
241 if !ok {
242 return
243 }
244 defer release()
245 rc := http.NewResponseController(w)
246 started := false
247 err := h.cfg.API.Tarball(ctx, p, owner, repo, ref, func(name string) io.Writer {
248 started = true
249 setArchiveHeaders(w, name)
250 return &idleWriter{w: w, rc: rc, idle: h.idle}
251 })
252 switch {
253 case err == nil:
254 case started:
255 // The status is out: the client sees a truncated archive. Log it.
256 log := zerowrap.FromCtx(ctx)
257 log.Warn().Err(err).Msg("github api archive cut short")
258 default:
259 h.fail(ctx, w, err)
260 }
261}
262
263// acquire takes an archive slot for the client, or answers 429 (the client
264// already has its share) or 503 (the server writes all it can) and reports
265// false. It never waits: an archive can take minutes.
266func (h *Handler) acquire(w http.ResponseWriter, client string) (release func(), ok bool) {
267 h.mu.Lock()
268 if h.clients[client] >= h.perCli {
269 h.mu.Unlock()
270 w.Header().Set("Retry-After", retryAfter)
271 jsonError(w, http.StatusTooManyRequests, "Too many archives in progress")
272 return nil, false
273 }
274 h.clients[client]++
275 h.mu.Unlock()
276 releaseClient := func() {
277 h.mu.Lock()
278 if h.clients[client]--; h.clients[client] <= 0 {
279 delete(h.clients, client)
280 }
281 h.mu.Unlock()
282 }
283 select {
284 case h.archives <- struct{}{}:
285 return func() { <-h.archives; releaseClient() }, true
286 default:
287 releaseClient()
288 w.Header().Set("Retry-After", retryAfter)
289 jsonError(w, http.StatusServiceUnavailable, "Server busy")
290 return nil, false
291 }
292}
293
294// idleWriter moves the write deadline of the connection forward before each
295// chunk, so that only a client that stops reading is cut. Where the response
296// writer cannot set a deadline, the writes are not bounded here.
297type idleWriter struct {
298 w io.Writer
299 rc *http.ResponseController
300 idle time.Duration
301}
302
303func (i *idleWriter) Write(b []byte) (int, error) {
304 _ = i.rc.SetWriteDeadline(time.Now().Add(i.idle))
305 return i.w.Write(b)
306}
307
308func setArchiveHeaders(w http.ResponseWriter, name string) {
309 w.Header().Set("Content-Type", "application/x-gzip")
310 w.Header().Set("Content-Disposition", "attachment; filename="+name)
311}
312
313// authenticate reads the credentials of the request. None is the anonymous
314// principal; credentials that are not valid count as a failure of the client
315// address.
316func (h *Handler) authenticate(r *http.Request, addr string) (domain.Principal, error) {
317 secret, present := credentials(r)
318 if !present {
319 return domain.Anonymous(), nil
320 }
321 p, err := h.cfg.Auth.Authenticate(r.Context(), secret)
322 if err != nil {
323 if errors.Is(err, domain.ErrUnauthenticated) {
324 h.limiter.Fail(r.Context(), addr)
325 }
326 return domain.Anonymous(), err
327 }
328 return p, nil
329}
330
331// credentials extracts the token of an Authorization header: the password of
332// Basic (the user is ignored, checkout sends x-access-token), or the value of
333// "Bearer" or "token".
334func credentials(r *http.Request) (secret string, present bool) {
335 auth := r.Header.Get("Authorization")
336 if auth == "" {
337 return "", false
338 }
339 if _, pass, ok := r.BasicAuth(); ok {
340 return pass, true
341 }
342 if scheme, tok, ok := strings.Cut(auth, " "); ok && (strings.EqualFold(scheme, "Bearer") || strings.EqualFold(scheme, "token")) {
343 return strings.TrimSpace(tok), true
344 }
345 return "", true
346}
347
348// fail answers an error of a use case. A repository the caller cannot read is
349// as not found as one that does not exist.
350func (h *Handler) fail(ctx context.Context, w http.ResponseWriter, err error) {
351 switch {
352 case errors.Is(err, domain.ErrNotFound), domain.IsInvalid(err):
353 notFound(w)
354 case errors.Is(err, domain.ErrResourceExhausted):
355 w.Header().Set("Retry-After", retryAfter)
356 jsonError(w, http.StatusServiceUnavailable, "Server busy")
357 case errors.Is(err, context.Canceled):
358 default:
359 log := zerowrap.FromCtx(ctx)
360 log.Error().Err(err).Msg("github api request failed")
361 jsonError(w, http.StatusInternalServerError, "Server Error")
362 }
363}
364
365func notFound(w http.ResponseWriter) {
366 w.Header().Set("Content-Type", "application/json; charset=utf-8")
367 w.WriteHeader(http.StatusNotFound)
368 _, _ = io.WriteString(w, notFoundBody)
369}
370
371func jsonError(w http.ResponseWriter, status int, msg string) {
372 w.Header().Set("Content-Type", "application/json; charset=utf-8")
373 w.WriteHeader(status)
374 _, _ = io.WriteString(w, `{"message":`+strconv.Quote(msg)+`,"documentation_url":"https://docs.github.com/rest"}`)
375}
376
377func writeJSON(w http.ResponseWriter, r *http.Request, body []byte) {
378 w.Header().Set("Content-Type", "application/json; charset=utf-8")
379 w.Header().Set("Content-Length", strconv.Itoa(len(body)))
380 w.WriteHeader(http.StatusOK)
381 if r.Method != http.MethodHead {
382 _, _ = w.Write(body)
383 }
384}