1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3// Package githubapi serves the subset of GitHub's REST API that the actions
4// running on ermite call under github.api_url (/-/github/api, ADR 0025 CI-2):
5//
6// GET /repos/{owner}/{repo}
7// GET /repos/{owner}/{repo}/hash-algorithm
8// GET /repos/{owner}/{repo}/tarball[/{ref}]
9//
10// Everything else under the prefix is GitHub's 404 JSON, so that an action
11// calling an endpoint of a feature ermite lacks fails as it does on a GitHub
12// instance without it, never silently succeeds. The auth is the forge's: a
13// token (stored, or the repository token of a job) as the password of Basic with
14// any user, or as "Bearer" or "token" credentials; no credentials is anonymous,
15// which reads public repositories.
16package githubapi
17
18import (
19 "cmp"
20 "context"
21 "errors"
22 "io"
23 "net/http"
24 "net/netip"
25 "strconv"
26 "strings"
27 "sync"
28 "time"
29
30 "github.com/bnema/zerowrap"
31
32 "bnema.dev/ermite/internal/clientlimit"
33 "bnema.dev/ermite/internal/domain"
34 "bnema.dev/ermite/internal/ports"
35)
36
37// Prefix is where the API is mounted.
38const Prefix = "/-/github/api"
39
40// Patterns are the ServeMux patterns the handler serves: its whole prefix.
41var Patterns = []string{Prefix + "/"}
42
43// notFoundBody is GitHub's body for an unknown endpoint or repository.
44const notFoundBody = `{"message":"Not Found","documentation_url":"https://docs.github.com/rest"}`
45
46// retryAfter is the Retry-After, in seconds, of an answer that asks the client to
47// come back (429 and 503).
48const retryAfter = "5"
49
50// Bounds on the archives written at once.
51const (
52 // DefaultMaxArchives is the archives written at once on the server.
53 DefaultMaxArchives = 16
54 // DefaultMaxArchivesPerClient is the archives written at once for one client
55 // (see clientlimit.Key). A runner host runs several jobs at once, each
56 // downloading its actions, and all come from one address; eight is half of
57 // the server's slots, so one host cannot take them all. Beyond it the answer
58 // is 429 with Retry-After: the runner waits and tries again.
59 DefaultMaxArchivesPerClient = 8
60 // DefaultWriteIdleTimeout is how long a write of the archive may wait for the
61 // client. The deadline moves forward with every chunk: an archive is bounded
62 // by its size, not by its duration, but a client that stops reading is cut.
63 DefaultWriteIdleTimeout = 2 * time.Minute
64)
65
66// Config configures a Handler.
67type Config struct {
68 Auth ports.AuthService
69 API ports.GitHubAPIService
70 // TrustedProxies are the networks allowed to set X-Forwarded-For.
71 TrustedProxies []netip.Prefix
72 // Limiter counts failed authentications; nil builds a default one.
73 Limiter ports.FailureLimiter
74 // MaxArchives bounds the archives written at once (each runs a git
75 // process); zero means DefaultMaxArchives. MaxArchivesPerClient bounds them
76 // per client; zero means DefaultMaxArchivesPerClient. WriteIdleTimeout is
77 // the idle write deadline; zero means DefaultWriteIdleTimeout.
78 MaxArchives int
79 MaxArchivesPerClient int
80 WriteIdleTimeout time.Duration
81}
82
83// Handler serves the REST subset.
84type Handler struct {
85 cfg Config
86 limiter ports.FailureLimiter
87 archives chan struct{}
88 perCli int
89 idle time.Duration
90
91 mu sync.Mutex // guards clients
92 clients map[string]int // archives being written, per clientlimit.Key
93}
94
95// New builds a Handler.
96func New(cfg Config) *Handler {
97 h := &Handler{cfg: cfg, limiter: cfg.Limiter}
98 if h.limiter == nil {
99 h.limiter = clientlimit.NewLimiter(nil)
100 }
101 n := cfg.MaxArchives
102 if n <= 0 {
103 n = DefaultMaxArchives
104 }
105 h.archives = make(chan struct{}, n)
106 h.perCli = cmp.Or(cfg.MaxArchivesPerClient, DefaultMaxArchivesPerClient)
107 h.idle = cmp.Or(cfg.WriteIdleTimeout, DefaultWriteIdleTimeout)
108 h.clients = make(map[string]int)
109 return h
110}
111
112// ServeHTTP implements http.Handler.
113func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
114 w.Header().Set("Cache-Control", "no-store")
115 w.Header().Set("X-Content-Type-Options", "nosniff")
116 rest := strings.TrimPrefix(r.URL.Path, Prefix)
117 kind, owner, repo, ref, ok := route(rest)
118 if !ok {
119 notFound(w)
120 return
121 }
122 if r.Method != http.MethodGet && r.Method != http.MethodHead {
123 w.Header().Set("Allow", "GET, HEAD")
124 jsonError(w, http.StatusMethodNotAllowed, "Method Not Allowed")
125 return
126 }
127
128 ctx := r.Context()
129 addr := clientlimit.ClientAddr(r, h.cfg.TrustedProxies)
130 if blocked, _ := h.limiter.Blocked(ctx, addr); blocked {
131 jsonError(w, http.StatusTooManyRequests, "API rate limit exceeded")
132 return
133 }
134 p, err := h.authenticate(r, addr)
135 if err != nil {
136 if errors.Is(err, domain.ErrUnauthenticated) {
137 jsonError(w, http.StatusUnauthorized, "Bad credentials")
138 return
139 }
140 log := zerowrap.FromCtx(ctx)
141 log.Error().Err(err).Msg("authenticate github api request")
142 jsonError(w, http.StatusInternalServerError, "Server Error")
143 return
144 }
145
146 switch kind {
147 case "repo":
148 body, err := h.cfg.API.GetRepo(ctx, p, owner, repo)
149 if err != nil {
150 h.fail(ctx, w, err)
151 return
152 }
153 writeJSON(w, r, body)
154 case "hash":
155 alg, err := h.cfg.API.HashAlgorithm(ctx, p, owner, repo)
156 if err != nil {
157 h.fail(ctx, w, err)
158 return
159 }
160 writeJSON(w, r, []byte(`{"hash_algorithm":`+strconv.Quote(alg)+`}`))
161 case "tarball":
162 h.tarball(w, r, p, owner, repo, ref, clientlimit.Key(addr))
163 }
164}
165
166// route recognises the three endpoints. ref is "" for /tarball.
167func route(rest string) (kind, owner, repo, ref string, ok bool) {
168 parts := strings.Split(strings.TrimPrefix(rest, "/"), "/")
169 if len(parts) < 3 || parts[0] != "repos" || parts[1] == "" || parts[2] == "" {
170 return "", "", "", "", false
171 }
172 owner, repo = parts[1], parts[2]
173 switch {
174 case len(parts) == 3:
175 return "repo", owner, repo, "", true
176 case len(parts) == 4 && parts[3] == "hash-algorithm":
177 return "hash", owner, repo, "", true
178 case len(parts) >= 4 && parts[3] == "tarball":
179 // The ref may hold slashes (feature/x): everything after tarball/.
180 return "tarball", owner, repo, strings.Join(parts[4:], "/"), true
181 }
182 return "", "", "", "", false
183}
184
185// tarball answers an archive request. A HEAD resolves the ref and sends the
186// headers of the archive, and runs no git. A GET streams the archive; it is not
187// a 302 to a codeload: ermite serves the bytes itself, which the runner follows
188// without a second credential.
189func (h *Handler) tarball(w http.ResponseWriter, r *http.Request, p domain.Principal, owner, repo, ref, client string) {
190 ctx := r.Context()
191 if r.Method == http.MethodHead {
192 name, err := h.cfg.API.TarballName(ctx, p, owner, repo, ref)
193 if err != nil {
194 h.fail(ctx, w, err)
195 return
196 }
197 setArchiveHeaders(w, name)
198 w.WriteHeader(http.StatusOK)
199 return
200 }
201 release, ok := h.acquire(w, client)
202 if !ok {
203 return
204 }
205 defer release()
206 rc := http.NewResponseController(w)
207 started := false
208 err := h.cfg.API.Tarball(ctx, p, owner, repo, ref, func(name string) io.Writer {
209 started = true
210 setArchiveHeaders(w, name)
211 return &idleWriter{w: w, rc: rc, idle: h.idle}
212 })
213 switch {
214 case err == nil:
215 case started:
216 // The status is out: the client sees a truncated archive. Log it.
217 log := zerowrap.FromCtx(ctx)
218 log.Warn().Err(err).Msg("github api archive cut short")
219 default:
220 h.fail(ctx, w, err)
221 }
222}
223
224// acquire takes an archive slot for the client, or answers 429 (the client
225// already has its share) or 503 (the server writes all it can) and reports
226// false. It never waits: an archive can take minutes.
227func (h *Handler) acquire(w http.ResponseWriter, client string) (release func(), ok bool) {
228 h.mu.Lock()
229 if h.clients[client] >= h.perCli {
230 h.mu.Unlock()
231 w.Header().Set("Retry-After", retryAfter)
232 jsonError(w, http.StatusTooManyRequests, "Too many archives in progress")
233 return nil, false
234 }
235 h.clients[client]++
236 h.mu.Unlock()
237 releaseClient := func() {
238 h.mu.Lock()
239 if h.clients[client]--; h.clients[client] <= 0 {
240 delete(h.clients, client)
241 }
242 h.mu.Unlock()
243 }
244 select {
245 case h.archives <- struct{}{}:
246 return func() { <-h.archives; releaseClient() }, true
247 default:
248 releaseClient()
249 w.Header().Set("Retry-After", retryAfter)
250 jsonError(w, http.StatusServiceUnavailable, "Server busy")
251 return nil, false
252 }
253}
254
255// idleWriter moves the write deadline of the connection forward before each
256// chunk, so that only a client that stops reading is cut. Where the response
257// writer cannot set a deadline, the writes are not bounded here.
258type idleWriter struct {
259 w io.Writer
260 rc *http.ResponseController
261 idle time.Duration
262}
263
264func (i *idleWriter) Write(b []byte) (int, error) {
265 _ = i.rc.SetWriteDeadline(time.Now().Add(i.idle))
266 return i.w.Write(b)
267}
268
269func setArchiveHeaders(w http.ResponseWriter, name string) {
270 w.Header().Set("Content-Type", "application/x-gzip")
271 w.Header().Set("Content-Disposition", "attachment; filename="+name)
272}
273
274// authenticate reads the credentials of the request. None is the anonymous
275// principal; credentials that are not valid count as a failure of the client
276// address.
277func (h *Handler) authenticate(r *http.Request, addr string) (domain.Principal, error) {
278 secret, present := credentials(r)
279 if !present {
280 return domain.Anonymous(), nil
281 }
282 p, err := h.cfg.Auth.Authenticate(r.Context(), secret)
283 if err != nil {
284 if errors.Is(err, domain.ErrUnauthenticated) {
285 h.limiter.Fail(r.Context(), addr)
286 }
287 return domain.Anonymous(), err
288 }
289 return p, nil
290}
291
292// credentials extracts the token of an Authorization header: the password of
293// Basic (the user is ignored, checkout sends x-access-token), or the value of
294// "Bearer" or "token".
295func credentials(r *http.Request) (secret string, present bool) {
296 auth := r.Header.Get("Authorization")
297 if auth == "" {
298 return "", false
299 }
300 if _, pass, ok := r.BasicAuth(); ok {
301 return pass, true
302 }
303 if scheme, tok, ok := strings.Cut(auth, " "); ok && (strings.EqualFold(scheme, "Bearer") || strings.EqualFold(scheme, "token")) {
304 return strings.TrimSpace(tok), true
305 }
306 return "", true
307}
308
309// fail answers an error of a use case. A repository the caller cannot read is
310// as not found as one that does not exist.
311func (h *Handler) fail(ctx context.Context, w http.ResponseWriter, err error) {
312 switch {
313 case errors.Is(err, domain.ErrNotFound), domain.IsInvalid(err):
314 notFound(w)
315 case errors.Is(err, domain.ErrResourceExhausted):
316 w.Header().Set("Retry-After", retryAfter)
317 jsonError(w, http.StatusServiceUnavailable, "Server busy")
318 case errors.Is(err, context.Canceled):
319 default:
320 log := zerowrap.FromCtx(ctx)
321 log.Error().Err(err).Msg("github api request failed")
322 jsonError(w, http.StatusInternalServerError, "Server Error")
323 }
324}
325
326func notFound(w http.ResponseWriter) {
327 w.Header().Set("Content-Type", "application/json; charset=utf-8")
328 w.WriteHeader(http.StatusNotFound)
329 _, _ = io.WriteString(w, notFoundBody)
330}
331
332func jsonError(w http.ResponseWriter, status int, msg string) {
333 w.Header().Set("Content-Type", "application/json; charset=utf-8")
334 w.WriteHeader(status)
335 _, _ = io.WriteString(w, `{"message":`+strconv.Quote(msg)+`,"documentation_url":"https://docs.github.com/rest"}`)
336}
337
338func writeJSON(w http.ResponseWriter, r *http.Request, body []byte) {
339 w.Header().Set("Content-Type", "application/json; charset=utf-8")
340 w.Header().Set("Content-Length", strconv.Itoa(len(body)))
341 w.WriteHeader(http.StatusOK)
342 if r.Method != http.MethodHead {
343 _, _ = w.Write(body)
344 }
345}