1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "context"
7 "crypto/ed25519"
8 "crypto/rand"
9 "errors"
10 "fmt"
11 "net"
12 "os"
13 "path/filepath"
14 "time"
15
16 "github.com/bnema/zerowrap"
17
18 "bnema.dev/ermite/internal/adapters/cilogs"
19 "bnema.dev/ermite/internal/adapters/cisummaries"
20 "bnema.dev/ermite/internal/adapters/ermitegit"
21 "bnema.dev/ermite/internal/adapters/eventbus"
22 "bnema.dev/ermite/internal/adapters/gitcli"
23 "bnema.dev/ermite/internal/adapters/githttp"
24 "bnema.dev/ermite/internal/adapters/githubapi"
25 "bnema.dev/ermite/internal/adapters/githubraw"
26 "bnema.dev/ermite/internal/adapters/gitremote"
27 "bnema.dev/ermite/internal/adapters/httpserver"
28 "bnema.dev/ermite/internal/adapters/kvstore"
29 "bnema.dev/ermite/internal/adapters/rpc"
30 "bnema.dev/ermite/internal/adapters/secretbox"
31 "bnema.dev/ermite/internal/adapters/sshsig"
32 "bnema.dev/ermite/internal/adapters/web"
33 "bnema.dev/ermite/internal/adapters/webauthn"
34 "bnema.dev/ermite/internal/clientlimit"
35 "bnema.dev/ermite/internal/config"
36 "bnema.dev/ermite/internal/domain"
37 "bnema.dev/ermite/internal/ports"
38 "bnema.dev/ermite/internal/usecase"
39)
40
41// maxReceiveBytes bounds the body of a push, both at the HTTP handler and in
42// Git (receive.maxInputSize).
43const maxReceiveBytes = githttp.DefaultMaxBodyBytes
44
45// options are the parts of the wiring that tests replace.
46type options struct {
47 // hookExe is the executable the hook scripts exec; empty means this
48 // process's own executable.
49 hookExe string
50 // maintenanceDelay overrides usecase.MaintenanceDelay when positive.
51 maintenanceDelay time.Duration
52 // maxBodyBytes overrides the limit of a push body when positive.
53 maxBodyBytes int64
54 // eventBuffer overrides usecase.DefaultEventBuffer when positive.
55 eventBuffer int
56 // revalidateEvery overrides usecase.DefaultRevalidateEvery when not zero.
57 revalidateEvery time.Duration
58 // archivesPerClient overrides the archives one client downloads at once
59 // when positive.
60 archivesPerClient int
61 // remote replaces the lister of github.com refs (tests never reach the
62 // network).
63 remoteLister ports.GitRemote
64 // workflowFetcher replaces the downloader of github.com reusable workflows
65 // (tests never reach the network).
66 workflowFetcher ports.WorkflowFetcher
67 // ci overrides the settings of the CI engine (tests shorten its
68 // durations).
69 ci func(*usecase.CIConfig)
70}
71
72// remote returns the lister of github.com refs: the test's, else def.
73func (o options) remote(def ports.GitRemote) ports.GitRemote {
74 if o.remoteLister != nil {
75 return o.remoteLister
76 }
77 return def
78}
79
80// workflows returns the downloader of github.com reusable workflows: the
81// test's, else def.
82func (o options) workflows(def ports.WorkflowFetcher) ports.WorkflowFetcher {
83 if o.workflowFetcher != nil {
84 return o.workflowFetcher
85 }
86 return def
87}
88
89// forge is the wired server: every adapter and use case, ready to serve.
90type forge struct {
91 cfg config.Config
92 store *kvstore.Store
93 server *httpserver.Server
94 hooks *githttp.HookServer
95 maint *usecase.Maintainer
96 git *gitcli.Git
97 // reader is the ermite-git reader when ERMITE_GIT_READER selects it.
98 reader *ermitegit.Adapter
99
100 // The use cases and the API are exposed for tests and for the phases
101 // that add the web.
102 Repos *usecase.Repos
103 Tokens *usecase.Tokens
104 Auth *usecase.Auth
105 Pushes *usecase.Pushes
106 Browse *usecase.Browse
107 // MergeRequests and SigningKey serve MergeRequestService and
108 // ServerService.GetSigningKey.
109 MergeRequests *usecase.MergeRequests
110 SigningKey *usecase.SigningKey
111 Sessions *usecase.Sessions
112 // Passkeys is nil when passkeys are not available (passkeysAvailable).
113 Passkeys *usecase.Passkeys
114 EventLog *usecase.Events
115 // CI is the CI engine: workflow trigger, planner, runner protocol and the
116 // owner side. It starts with the server and stops with it.
117 CI *usecase.CI
118 // Settings are the secrets, variables and environments of CI.
119 Settings *usecase.CISettings
120 API *rpc.API
121 Bus ports.EventBus
122}
123
124// newSigningSeed makes the seed of a new ed25519 signing key.
125func newSigningSeed() ([]byte, error) {
126 seed := make([]byte, ed25519.SeedSize)
127 if _, err := rand.Read(seed); err != nil {
128 return nil, err
129 }
130 return seed, nil
131}
132
133// serverVersion is the version GetVersion reports: the module version of the
134// build, or "devel" for an untagged one.
135func serverVersion() string {
136 if v := buildVersion(); v != "(devel)" {
137 return v
138 }
139 return "devel"
140}
141
142// newForge opens the metadata store, resets run/, checks git, reconciles the repositories,
143// writes the hook scripts, opens the hook socket and builds the HTTP server.
144// The data directory must be prepared (prepareDataDir). Close it after the
145// server stopped.
146func newForge(ctx context.Context, cfg config.Config, opts options) (_ *forge, err error) {
147 log := zerowrap.FromCtx(ctx)
148 store, err := kvstore.Open(ctx, filepath.Join(cfg.DataDir, "kv"))
149 if err != nil {
150 return nil, err
151 }
152 f := &forge{cfg: cfg, store: store}
153 defer func() {
154 if err != nil {
155 f.Close(context.WithoutCancel(ctx))
156 }
157 }()
158 // The kv store holds the directory lock: only now is run/ ours to reset.
159 // The hook scripts and the socket are written into it afterwards.
160 if err := resetRunDir(cfg.DataDir); err != nil {
161 return nil, err
162 }
163
164 path := os.Getenv("PATH")
165 if path == "" {
166 path = "/usr/local/bin:/usr/bin:/bin"
167 }
168 maxBody := maxReceiveBytes
169 if opts.maxBodyBytes > 0 {
170 maxBody = opts.maxBodyBytes
171 }
172 git := gitcli.New(gitcli.Config{Bin: cfg.Git, DataDir: cfg.DataDir, Path: path, MaxReceiveBytes: maxBody})
173 f.git = git
174 gitVersion, err := git.CheckVersion(ctx)
175 if err != nil {
176 return nil, fmt.Errorf("%s=%q: %w", config.EnvGit, cfg.Git, err)
177 }
178 log.Info().Str("git", gitVersion.String()).Msg("git checked")
179 if _, err := git.HTTPBackend(ctx, false); err != nil {
180 return nil, fmt.Errorf("%s=%q: %w", config.EnvGit, cfg.Git, err)
181 }
182
183 clock := systemClock{}
184 f.Bus = eventbus.New()
185 f.Repos = usecase.NewRepos(store, git, f.Bus, clock)
186 report, err := f.Repos.Reconcile(ctx)
187 if err != nil {
188 return nil, fmt.Errorf("reconcile repositories: %w", err)
189 }
190 log.Info().
191 Int("registered", len(report.Registered)).
192 Int("missing_dirs", len(report.MissingDirs)).
193 Int("invalid_dirs", len(report.InvalidDirs)).
194 Msg("repositories reconciled")
195
196 f.Tokens = usecase.NewTokens(store.Tokens(), clock, store.Sessions())
197 f.Auth = usecase.NewAuth(cfg.AdminTokenHash, store.Tokens(), clock).WithRepoTokens(store.CI())
198 f.maint = usecase.NewMaintainer(ctx, git, opts.maintenanceDelay)
199 f.Pushes = usecase.NewPushes(store, f.Repos, git, f.Bus, clock, f.maint, f.Auth)
200
201 exe := opts.hookExe
202 if exe == "" {
203 if exe, err = os.Executable(); err != nil {
204 return nil, fmt.Errorf("find the ermite executable for the hooks: %w", err)
205 }
206 }
207 if err := githttp.WriteHookScripts(gitcli.HooksDir(cfg.DataDir), exe); err != nil {
208 return nil, err
209 }
210 f.hooks = githttp.NewHookServer(filepath.Join(cfg.DataDir, "run", "hook.sock"), f.Pushes)
211 if err := f.hooks.Listen(ctx); err != nil {
212 return nil, err
213 }
214
215 var reader ports.RepoReader = git
216 if cfg.GitReader == config.GitReaderErmiteGit {
217 f.reader = ermitegit.New(ermitegit.Config{
218 Fallback: git,
219 RepoDir: func(name domain.RepoName) string { return gitcli.RepoDir(cfg.DataDir, name) },
220 })
221 git.OnTrash(f.reader.Forget)
222 f.maint.WithFreshener(f.reader)
223 f.Pushes.WithFreshener(f.reader)
224 reader = f.reader
225 }
226 log.Info().Str("reader", cfg.GitReader).Msg("git reader selected")
227 f.Browse = usecase.NewBrowse(store, reader)
228
229 // Secrets, variables and environments (ADR 0019, CI-3). With a key the
230 // secrets are sealed under it, and the server refuses to start when the
231 // key is not the one they were sealed with; without one they cannot be set
232 // and jobs receive none.
233 var sealer ports.SecretSealer // a nil *secretbox.Box must not become a non-nil interface
234 if cfg.SecretsKey != nil {
235 box, err := secretbox.New(cfg.SecretsKey)
236 if err != nil {
237 return nil, fmt.Errorf("%s: %w", config.EnvSecretsKey, err)
238 }
239 sealer = box
240 }
241 f.Settings, err = usecase.NewCISettings(ctx, store, store.Settings(), sealer, clock)
242 if err != nil {
243 return nil, fmt.Errorf("%s: %w", config.EnvSecretsKey, err)
244 }
245
246 // CI (ADR 0025): the engine reads workflows through the reader, and plans
247 // them in the background; a push only queues its refs.
248 committer := domain.Identity{Name: cfg.CommitterName, Email: cfg.CommitterEmail}
249 ciCfg := usecase.CIConfig{
250 Store: store.CI(), Logs: cilogs.New(cilogs.Dir(cfg.DataDir)), Summaries: cisummaries.New(cisummaries.Dir(cfg.DataDir)), Meta: store, Repos: git, Reader: reader,
251 Git: git, Diff: git, Bus: f.Bus, Clock: clock, Auth: f.Auth,
252 Owner: cfg.Owner, PublicURL: cfg.PublicURL, Retention: cfg.CIRetention,
253 PollTimeout: cfg.CIPollTimeout, Archive: git,
254 Remote: opts.remote(gitremote.New(gitremote.Config{
255 Bin: cfg.Git, Path: path, Home: filepath.Join(cfg.DataDir, "run", "git-home"),
256 Proxy: gitremote.ProxyEnv(os.Getenv),
257 })),
258 Tokens: store.Tokens(),
259 RefTips: store.CI(),
260 Settings: f.Settings,
261 Workflows: opts.workflows(githubraw.New(githubraw.Config{})),
262 }
263 f.pullRequests(&ciCfg, store, git, committer)
264 if opts.ci != nil {
265 opts.ci(&ciCfg)
266 }
267 f.CI = usecase.NewCI(ctx, ciCfg)
268 f.Pushes.WithWorkflowTrigger(f.CI).WithWorkflowGuard(git).WithRefTips(store.CI())
269 f.Repos.WithCIRunCleaner(f.CI)
270 f.CI.Start()
271 f.EventLog = usecase.NewEvents(ctx, f.Repos, f.Bus, clock, usecase.EventsConfig{
272 Size: opts.eventBuffer, Auth: f.Auth, RevalidateEvery: opts.revalidateEvery,
273 })
274
275 // The instance signing key (ADR 0022): generated on the first start.
276 seed, err := store.SigningSeed(ctx, newSigningSeed, clock.Now().UTC())
277 if err != nil {
278 return nil, err
279 }
280 signer, err := sshsig.New(seed)
281 if err != nil {
282 return nil, err
283 }
284 f.SigningKey = usecase.NewSigningKey(signer, committer)
285 f.MergeRequests = usecase.NewMergeRequests(usecase.MergeRequestsConfig{
286 Meta: store, Store: store.MergeRequests(), Repos: git, Git: git, Signer: signer,
287 Committer: committer, Bus: f.Bus, Clock: clock, Maint: f.maint,
288 CI: f.CI, Checks: f.CI, PullRequests: f.CI, RefTips: store.CI(),
289 })
290 f.CI.SetPullGate(f.MergeRequests)
291 log.Info().Str("fingerprint", signer.PublicKey().Fingerprint).Msg("commit signing key ready")
292
293 f.Sessions, err = usecase.NewSessions(ctx, store.Sessions(), clock, cfg.PublicURL)
294 if err != nil {
295 return nil, err
296 }
297
298 // Owner passkeys (ADR 0026): built only when the public URL allows them.
299 // When it does, a verifier that cannot be built is fatal.
300 if ok, reason := passkeysAvailable(cfg.PublicURL); ok {
301 verifier, err := webauthn.New(cfg.PublicURL)
302 if err != nil {
303 return nil, fmt.Errorf("passkeys: %w", err)
304 }
305 f.Passkeys = usecase.NewPasskeys(store.Passkeys(), verifier, f.Sessions, clock)
306 log.Info().Str("public_url", cfg.PublicURL).Msg("passkeys enabled")
307 } else {
308 log.Info().Str("reason", reason).Msg("passkeys disabled")
309 }
310
311 // One budget of failed authentications per client, for Git, the API and
312 // the web login.
313 limiter := clientlimit.NewLimiter(nil)
314 // A nil *usecase.Passkeys must not become a non-nil interface: the API
315 // tests for nil to answer failed_precondition and to leave the feature out.
316 var apiPasskeys ports.PasskeyService
317 if f.Passkeys != nil {
318 apiPasskeys = f.Passkeys
319 }
320 f.API = rpc.New(rpc.Config{
321 Auth: f.Auth,
322 Repos: f.Repos,
323 Tokens: f.Tokens,
324 Browse: f.Browse,
325 Events: f.EventLog,
326 MergeRequests: f.MergeRequests,
327 SigningKey: f.SigningKey,
328 Sessions: f.Sessions,
329 Passkeys: apiPasskeys,
330 Runner: f.CI,
331 Runs: f.CI,
332 Secrets: f.Settings,
333 Variables: f.Settings,
334 Environments: f.Settings,
335 Version: serverVersion(),
336 TrustedProxies: cfg.TrustedProxies,
337 Limiter: limiter,
338 })
339
340 gh := githttp.New(githttp.Config{
341 Auth: f.Auth,
342 Repos: f.Repos,
343 Pushes: f.Pushes,
344 Backend: git,
345 Owner: cfg.Owner,
346 Hooks: f.hooks,
347 TrustedProxies: cfg.TrustedProxies,
348 Limiter: limiter,
349 MaxBodyBytes: maxBody,
350 })
351 routes := make([]httpserver.Route, 0, 2*len(githttp.Patterns)+len(githubapi.Patterns)+len(web.Patterns))
352 for _, p := range githttp.Patterns {
353 routes = append(routes, httpserver.Route{Pattern: p, Handler: gh})
354 }
355 // The owner form of the clone URL (github.repository), which GitHub's
356 // actions use. config.Owner follows the rules of repository names, so it is
357 // a literal segment that never overlaps /-/.
358 for _, p := range githttp.OwnerPatterns(cfg.Owner) {
359 routes = append(routes, httpserver.Route{Pattern: p, Handler: gh})
360 }
361 // The REST subset the actions call on github.api_url.
362 api := githubapi.New(githubapi.Config{Auth: f.Auth, API: f.CI, TrustedProxies: cfg.TrustedProxies, Limiter: limiter, MaxArchivesPerClient: opts.archivesPerClient})
363 for _, p := range githubapi.Patterns {
364 routes = append(routes, httpserver.Route{Pattern: p, Handler: api})
365 }
366 warnSessionsNeedHTTPS(ctx, cfg.PublicURL)
367 warnSharedClientKey(ctx, cfg.TrustedProxies, githttp.DefaultMaxPerClient)
368 site, err := newWeb(cfg, f, limiter)
369 if err != nil {
370 return nil, fmt.Errorf("web: %w", err)
371 }
372 routes = append(routes, webRoutes(site)...)
373 f.server = httpserver.New(httpserver.Config{
374 Addr: cfg.Addr,
375 PublicURL: cfg.PublicURL,
376 HealthChecks: []httpserver.Check{store.Check, httpserver.DirWritable(cfg.DataDir)},
377 Routes: routes,
378 API: f.API.Handler(),
379 // Event streams never end by themselves: close them when the
380 // shutdown begins, or it would wait for them until its timeout.
381 OnShutdown: []func(){func() { f.EventLog.Close(context.WithoutCancel(ctx)) }},
382 })
383 return f, nil
384}
385
386// Run listens on the configured address and serves until ctx ends.
387func (f *forge) Run(ctx context.Context) error {
388 ln, err := net.Listen("tcp", f.cfg.Addr)
389 if err != nil {
390 return fmt.Errorf("listen on %s: %w", f.cfg.Addr, err)
391 }
392 return f.Serve(ctx, ln)
393}
394
395// Serve serves HTTP on ln and the hook socket until ctx ends. The hook socket
396// outlives the HTTP server: requests still in flight while it drains are
397// pushes whose hooks need it. A failure of either stops both.
398func (f *forge) Serve(ctx context.Context, ln net.Listener) error {
399 ctx, cancel := context.WithCancel(ctx)
400 defer cancel()
401 hookCtx, stopHooks := context.WithCancel(context.WithoutCancel(ctx))
402 defer stopHooks()
403
404 hookErr := make(chan error, 1)
405 go func() {
406 err := f.hooks.Serve(hookCtx)
407 cancel() // a dead hook socket would reject every push: stop serving
408 hookErr <- err
409 }()
410
411 httpErr := f.server.Serve(ctx, ln)
412 stopHooks()
413 return errors.Join(httpErr, <-hookErr)
414}
415
416// Close releases what newForge opened. It is safe to call twice. Call it after
417// the HTTP server stopped: it stops the event log and the long-lived Git
418// processes of the read side, which requests in flight may still use.
419func (f *forge) Close(ctx context.Context) {
420 log := zerowrap.FromCtx(ctx)
421 if f.hooks != nil {
422 f.hooks.Close(ctx)
423 }
424 if f.CI != nil {
425 f.CI.Close(ctx)
426 }
427 if f.EventLog != nil {
428 f.EventLog.Close(ctx)
429 }
430 if f.reader != nil {
431 f.reader.Close()
432 }
433 if f.git != nil {
434 if err := f.git.Close(ctx); err != nil {
435 log.Error().Err(err).Msg("close git reader")
436 }
437 }
438 if f.maint != nil {
439 f.maint.Close(ctx)
440 }
441 if f.store != nil {
442 if err := f.store.Close(ctx); err != nil {
443 log.Error().Err(err).Msg("close metadata store")
444 }
445 f.store = nil
446 }
447}
448
449// pullRequests gives the CI the merge requests, the refs/pull refs and the
450// identity of the test merge commits (CI-3, D5).
451func (f *forge) pullRequests(cfg *usecase.CIConfig, store *kvstore.Store, git *gitcli.Git, committer domain.Identity) {
452 cfg.MRs, cfg.Pull, cfg.Committer = store.MergeRequests(), git, committer
453 if f.reader != nil {
454 cfg.Fresh = f.reader // a nil *ermitegit.Adapter must not become a non-nil interface
455 }
456}