1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3// Package gitremote lists the refs of repositories on github.com with
4// `git ls-remote`, for the resolution of actions (ports.GitRemote). It sends no
5// credentials, never prompts, and bounds the time and the output of git.
6package gitremote
7
8import (
9 "bytes"
10 "context"
11 "errors"
12 "fmt"
13 "os/exec"
14 "strings"
15 "time"
16
17 "bnema.dev/ermite/internal/domain"
18 "bnema.dev/ermite/internal/ports"
19)
20
21// Defaults of the bounds.
22const (
23 // DefaultTimeout is how long ListRefs may take in all, its attempts
24 // included. It is below the 30 s the runner gives a call.
25 DefaultTimeout = 20 * time.Second
26 // DefaultMaxOutput is the most bytes of ls-remote output read: a repository
27 // with 100,000 tags lists about 8 MB. More is refused as unavailable.
28 DefaultMaxOutput = 16 << 20
29 // DefaultBaseURL is where the repositories are.
30 DefaultBaseURL = "https://github.com"
31)
32
33// Config configures a Remote.
34type Config struct {
35 // Bin is the git executable. Required.
36 Bin string
37 // Path is the PATH given to git, which it uses to find its helpers.
38 Path string
39 // Home is the HOME of git; it should be an empty directory the process
40 // owns, so that no user configuration or credential is read.
41 Home string
42 // BaseURL, Timeout and MaxOutput default to the constants above; tests
43 // replace them.
44 BaseURL string
45 Timeout time.Duration
46 MaxOutput int
47 // AllowedProtocols is GIT_ALLOW_PROTOCOL: "https" by default.
48 AllowedProtocols string
49 // Proxy is the proxy environment forwarded to git, as NAME=value pairs: see
50 // ProxyEnv. Nothing else of the environment of ermite reaches git.
51 Proxy []string
52}
53
54// proxyVars are the variables git (through libcurl) reads to reach github.com
55// through a proxy. They are the only ones of the environment of ermite that
56// the ls-remote processes get.
57var proxyVars = []string{
58 "HTTPS_PROXY", "HTTP_PROXY", "NO_PROXY", "ALL_PROXY",
59 "https_proxy", "http_proxy", "no_proxy", "all_proxy",
60}
61
62// ProxyEnv picks the proxy variables from an environment lookup (os.Getenv),
63// for Config.Proxy. Empty ones are left out.
64func ProxyEnv(get func(string) string) []string {
65 var out []string
66 for _, name := range proxyVars {
67 if v := get(name); v != "" {
68 out = append(out, name+"="+v)
69 }
70 }
71 return out
72}
73
74// Remote implements ports.GitRemote over the git binary.
75type Remote struct{ cfg Config }
76
77var _ ports.GitRemote = (*Remote)(nil)
78
79// New builds a Remote.
80func New(cfg Config) *Remote {
81 if cfg.BaseURL == "" {
82 cfg.BaseURL = DefaultBaseURL
83 }
84 if cfg.Timeout <= 0 {
85 cfg.Timeout = DefaultTimeout
86 }
87 if cfg.MaxOutput <= 0 {
88 cfg.MaxOutput = DefaultMaxOutput
89 }
90 if cfg.AllowedProtocols == "" {
91 cfg.AllowedProtocols = "https"
92 }
93 return &Remote{cfg: cfg}
94}
95
96// validSegment reports whether s can be an owner or a repository name on
97// GitHub: letters, digits, '.', '_' and '-', not starting with '-' or '.'.
98func validSegment(s string) bool {
99 if s == "" || len(s) > 100 || s[0] == '-' || s[0] == '.' {
100 return false
101 }
102 for _, r := range s {
103 switch {
104 case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '-':
105 default:
106 return false
107 }
108 }
109 return true
110}
111
112// limitWriter keeps at most max bytes and records that more came.
113type limitWriter struct {
114 buf bytes.Buffer
115 max int
116 exceeded bool
117 cancel context.CancelFunc
118}
119
120func (w *limitWriter) Write(p []byte) (int, error) {
121 if w.buf.Len()+len(p) > w.max {
122 w.exceeded = true
123 w.cancel()
124 return 0, errors.New("output limit reached")
125 }
126 return w.buf.Write(p)
127}
128
129// attempts is how many times a ls-remote that timed out is run: a connection
130// that github.com never answered (a lost SYN) is often not repeated by a new
131// one, which may also reach another address.
132const attempts = 2
133
134// ListRefs lists the branches and tags of owner/repo, with the peeled commit
135// of the annotated tags. Timeout bounds the whole call, attempts and clean-up
136// included: the time to clean up after an attempt (kill, WaitDelay) is
137// reserved before it runs, and a ls-remote that timed out is run once more
138// with what is left, at most half. The cancellation of ctx is returned as it
139// is, not retried.
140func (r *Remote) ListRefs(ctx context.Context, owner, repo string) ([]ports.RemoteRef, error) {
141 if !validSegment(owner) || !validSegment(repo) {
142 return nil, domain.ErrNotFound
143 }
144 deadline := time.Now().Add(r.cfg.Timeout)
145 cleanup := max(min(2*time.Second, r.cfg.Timeout/8), time.Millisecond)
146 ran := 0
147 for ran < attempts {
148 run := min(r.cfg.Timeout/attempts, time.Until(deadline)-cleanup)
149 if run <= 0 {
150 break
151 }
152 ran++
153 refs, timedOut, err := r.listOnce(ctx, owner, repo, run, cleanup)
154 switch {
155 case ctx.Err() != nil:
156 return nil, ctx.Err() // the caller gave up
157 case !timedOut:
158 return refs, err
159 }
160 }
161 return nil, fmt.Errorf("%w: git ls-remote %s/%s did not answer within %s (%d attempts)", domain.ErrUnavailable, owner, repo, r.cfg.Timeout, ran)
162}
163
164// listOnce runs one git ls-remote for at most d, then cleans up for at most
165// cleanup. timedOut tells that d ran out and parent did not.
166func (r *Remote) listOnce(parent context.Context, owner, repo string, d, cleanup time.Duration) (refs []ports.RemoteRef, timedOut bool, _ error) {
167 ctx, cancel := context.WithTimeout(parent, d)
168 defer cancel()
169 out := &limitWriter{max: r.cfg.MaxOutput, cancel: cancel}
170 var stderr bytes.Buffer
171 url := strings.TrimSuffix(r.cfg.BaseURL, "/") + "/" + owner + "/" + repo
172 cmd := exec.Command(r.cfg.Bin,
173 "-c", "credential.helper=", "-c", "core.askPass=", "-c", "http.followRedirects=false",
174 "ls-remote", "--", url, "refs/heads/*", "refs/tags/*")
175 cmd.Env = []string{
176 "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", "HOME=" + r.cfg.Home,
177 "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=", "SSH_ASKPASS=", "GCM_INTERACTIVE=never",
178 "GIT_ALLOW_PROTOCOL=" + r.cfg.AllowedProtocols, "LC_ALL=C", "PATH=" + r.cfg.Path,
179 }
180 cmd.Env = append(cmd.Env, r.cfg.Proxy...)
181 cmd.Dir = r.cfg.Home
182 cmd.Stdout, cmd.Stderr = out, &limitedBuffer{b: &stderr, max: 4096}
183 // The backstop for a helper that escaped the kill and holds the pipes: the
184 // time reserved for it by ListRefs.
185 cmd.WaitDelay = cleanup
186 err := run(ctx, cmd)
187 switch {
188 case out.exceeded:
189 return nil, false, fmt.Errorf("%w: git ls-remote %s/%s listed more than %d bytes", domain.ErrUnavailable, owner, repo, r.cfg.MaxOutput)
190 case err != nil && ctx.Err() != nil:
191 return nil, true, nil
192 case err != nil:
193 msg := stderr.String()
194 if notFound(msg) {
195 return nil, false, domain.ErrNotFound
196 }
197 if firstLine(msg) == "" {
198 msg = err.Error()
199 }
200 return nil, false, fmt.Errorf("%w: git ls-remote %s/%s: %v", domain.ErrUnavailable, owner, repo, firstLine(msg))
201 }
202 return parse(out.buf.String()), false, nil
203}
204
205// notFound recognises what git says when github.com answers 404 or asks for
206// credentials, which it does for a repository that does not exist as for a
207// private one, and the prompt being disabled.
208func notFound(stderr string) bool {
209 for _, s := range []string{"Repository not found", "terminal prompts disabled", "could not read Username", "Authentication failed", "returned error: 404"} {
210 if strings.Contains(stderr, s) {
211 return true
212 }
213 }
214 return false
215}
216
217func firstLine(s string) string {
218 s = strings.TrimSpace(s)
219 if i := strings.IndexByte(s, '\n'); i >= 0 {
220 s = s[:i]
221 }
222 return s
223}
224
225// limitedBuffer keeps the first max bytes written and drops the rest.
226type limitedBuffer struct {
227 b *bytes.Buffer
228 max int
229}
230
231func (l *limitedBuffer) Write(p []byte) (int, error) {
232 if room := l.max - l.b.Len(); room > 0 {
233 l.b.Write(p[:min(room, len(p))])
234 }
235 return len(p), nil
236}
237
238// parse reads the lines "<id>\t<ref>" of ls-remote. A "<ref>^{}" line gives
239// the peeled commit of the annotated tag listed before it.
240func parse(out string) []ports.RemoteRef {
241 var refs []ports.RemoteRef
242 index := map[string]int{}
243 for line := range strings.SplitSeq(out, "\n") {
244 id, name, ok := strings.Cut(line, "\t")
245 if !ok || len(id) != domain.ObjectIDLen {
246 continue
247 }
248 if base, peeled := strings.CutSuffix(name, "^{}"); peeled {
249 if i, ok := index[base]; ok {
250 refs[i].Peeled = id
251 }
252 continue
253 }
254 index[name] = len(refs)
255 refs = append(refs, ports.RemoteRef{Name: name, ID: id})
256 }
257 return refs
258}