1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "context"
7 "crypto/ed25519"
8 "crypto/rand"
9 "errors"
10 "fmt"
11 "net"
12 "os"
13 "path/filepath"
14 "time"
15
16 "github.com/bnema/zerowrap"
17
18 "bnema.dev/ermite/internal/adapters/cilogs"
19 "bnema.dev/ermite/internal/adapters/cisummaries"
20 "bnema.dev/ermite/internal/adapters/ermitegit"
21 "bnema.dev/ermite/internal/adapters/eventbus"
22 "bnema.dev/ermite/internal/adapters/gitcli"
23 "bnema.dev/ermite/internal/adapters/githttp"
24 "bnema.dev/ermite/internal/adapters/githubapi"
25 "bnema.dev/ermite/internal/adapters/githubraw"
26 "bnema.dev/ermite/internal/adapters/gitremote"
27 "bnema.dev/ermite/internal/adapters/httpserver"
28 "bnema.dev/ermite/internal/adapters/kvstore"
29 "bnema.dev/ermite/internal/adapters/rpc"
30 "bnema.dev/ermite/internal/adapters/secretbox"
31 "bnema.dev/ermite/internal/adapters/sshsig"
32 "bnema.dev/ermite/internal/adapters/web"
33 "bnema.dev/ermite/internal/adapters/webauthn"
34 "bnema.dev/ermite/internal/clientlimit"
35 "bnema.dev/ermite/internal/config"
36 "bnema.dev/ermite/internal/domain"
37 "bnema.dev/ermite/internal/ports"
38 "bnema.dev/ermite/internal/usecase"
39)
40
41// maxReceiveBytes bounds the body of a push, both at the HTTP handler and in
42// Git (receive.maxInputSize).
43const maxReceiveBytes = githttp.DefaultMaxBodyBytes
44
45// options are the parts of the wiring that tests replace.
46type options struct {
47 // hookExe is the executable the hook scripts exec; empty means this
48 // process's own executable.
49 hookExe string
50 // maintenanceDelay overrides usecase.MaintenanceDelay when positive.
51 maintenanceDelay time.Duration
52 // maxBodyBytes overrides the limit of a push body when positive.
53 maxBodyBytes int64
54 // eventBuffer overrides usecase.DefaultEventBuffer when positive.
55 eventBuffer int
56 // revalidateEvery overrides usecase.DefaultRevalidateEvery when not zero.
57 revalidateEvery time.Duration
58 // archivesPerClient overrides the archives one client downloads at once
59 // when positive.
60 archivesPerClient int
61 // remote replaces the lister of github.com refs (tests never reach the
62 // network).
63 remoteLister ports.GitRemote
64 // workflowFetcher replaces the downloader of github.com reusable workflows
65 // (tests never reach the network).
66 workflowFetcher ports.WorkflowFetcher
67 // ci overrides the settings of the CI engine (tests shorten its
68 // durations).
69 ci func(*usecase.CIConfig)
70}
71
72// remote returns the lister of github.com refs: the test's, else def.
73func (o options) remote(def ports.GitRemote) ports.GitRemote {
74 if o.remoteLister != nil {
75 return o.remoteLister
76 }
77 return def
78}
79
80// workflows returns the downloader of github.com reusable workflows: the
81// test's, else def.
82func (o options) workflows(def ports.WorkflowFetcher) ports.WorkflowFetcher {
83 if o.workflowFetcher != nil {
84 return o.workflowFetcher
85 }
86 return def
87}
88
89// forge is the wired server: every adapter and use case, ready to serve.
90type forge struct {
91 cfg config.Config
92 store *kvstore.Store
93 server *httpserver.Server
94 hooks *githttp.HookServer
95 maint *usecase.Maintainer
96 git *gitcli.Git
97 // reader is the ermite-git reader when ERMITE_GIT_READER selects it.
98 reader *ermitegit.Adapter
99
100 // The use cases and the API are exposed for tests and for the phases
101 // that add the web.
102 Repos *usecase.Repos
103 Tokens *usecase.Tokens
104 Auth *usecase.Auth
105 Pushes *usecase.Pushes
106 Browse *usecase.Browse
107 // MergeRequests and SigningKey serve MergeRequestService and
108 // ServerService.GetSigningKey.
109 MergeRequests *usecase.MergeRequests
110 SigningKey *usecase.SigningKey
111 Sessions *usecase.Sessions
112 // Passkeys is nil when passkeys are not available (passkeysAvailable).
113 Passkeys *usecase.Passkeys
114 EventLog *usecase.Events
115 // CI is the CI engine: workflow trigger, planner, runner protocol and the
116 // owner side. It starts with the server and stops with it.
117 CI *usecase.CI
118 // Settings are the secrets, variables and environments of CI.
119 Settings *usecase.CISettings
120 API *rpc.API
121 Bus ports.EventBus
122}
123
124// newSigningSeed makes the seed of a new ed25519 signing key.
125func newSigningSeed() ([]byte, error) {
126 seed := make([]byte, ed25519.SeedSize)
127 if _, err := rand.Read(seed); err != nil {
128 return nil, err
129 }
130 return seed, nil
131}
132
133// serverVersion is the version GetVersion reports: the module version of the
134// build, or "devel" for an untagged one.
135func serverVersion() string {
136 if v := buildVersion(); v != "(devel)" {
137 return v
138 }
139 return "devel"
140}
141
142// newForge opens the metadata store, resets run/, checks git, reconciles the repositories,
143// writes the hook scripts, opens the hook socket and builds the HTTP server.
144// The data directory must be prepared (prepareDataDir). Close it after the
145// server stopped.
146func newForge(ctx context.Context, cfg config.Config, opts options) (_ *forge, err error) {
147 log := zerowrap.FromCtx(ctx)
148 store, err := kvstore.Open(ctx, filepath.Join(cfg.DataDir, "kv"))
149 if err != nil {
150 return nil, err
151 }
152 f := &forge{cfg: cfg, store: store}
153 defer func() {
154 if err != nil {
155 f.Close(context.WithoutCancel(ctx))
156 }
157 }()
158 // The kv store holds the directory lock: only now is run/ ours to reset.
159 // The hook scripts and the socket are written into it afterwards.
160 if err := resetRunDir(cfg.DataDir); err != nil {
161 return nil, err
162 }
163
164 path := os.Getenv("PATH")
165 if path == "" {
166 path = "/usr/local/bin:/usr/bin:/bin"
167 }
168 maxBody := maxReceiveBytes
169 if opts.maxBodyBytes > 0 {
170 maxBody = opts.maxBodyBytes
171 }
172 git := gitcli.New(gitcli.Config{Bin: cfg.Git, DataDir: cfg.DataDir, Path: path, MaxReceiveBytes: maxBody})
173 f.git = git
174 gitVersion, err := git.CheckVersion(ctx)
175 if err != nil {
176 return nil, fmt.Errorf("%s=%q: %w", config.EnvGit, cfg.Git, err)
177 }
178 log.Info().Str("git", gitVersion.String()).Msg("git checked")
179 if _, err := git.HTTPBackend(ctx, false); err != nil {
180 return nil, fmt.Errorf("%s=%q: %w", config.EnvGit, cfg.Git, err)
181 }
182
183 clock := systemClock{}
184 f.Bus = eventbus.New()
185 f.Repos = usecase.NewRepos(store, git, f.Bus, clock)
186 report, err := f.Repos.Reconcile(ctx)
187 if err != nil {
188 return nil, fmt.Errorf("reconcile repositories: %w", err)
189 }
190 log.Info().
191 Int("registered", len(report.Registered)).
192 Int("missing_dirs", len(report.MissingDirs)).
193 Int("invalid_dirs", len(report.InvalidDirs)).
194 Msg("repositories reconciled")
195
196 f.Tokens = usecase.NewTokens(store.Tokens(), clock, store.Sessions())
197 f.Auth = usecase.NewAuth(cfg.AdminTokenHash, store.Tokens(), clock).WithRepoTokens(store.CI())
198 f.maint = usecase.NewMaintainer(ctx, git, opts.maintenanceDelay)
199 f.Pushes = usecase.NewPushes(store, f.Repos, git, f.Bus, clock, f.maint, f.Auth)
200
201 exe := opts.hookExe
202 if exe == "" {
203 if exe, err = os.Executable(); err != nil {
204 return nil, fmt.Errorf("find the ermite executable for the hooks: %w", err)
205 }
206 }
207 if err := githttp.WriteHookScripts(gitcli.HooksDir(cfg.DataDir), exe); err != nil {
208 return nil, err
209 }
210 f.hooks = githttp.NewHookServer(filepath.Join(cfg.DataDir, "run", "hook.sock"), f.Pushes)
211 if err := f.hooks.Listen(ctx); err != nil {
212 return nil, err
213 }
214
215 var reader ports.RepoReader = git
216 if cfg.GitReader == config.GitReaderErmiteGit {
217 f.reader = ermitegit.New(ermitegit.Config{
218 Fallback: git,
219 RepoDir: func(name domain.RepoName) string { return gitcli.RepoDir(cfg.DataDir, name) },
220 })
221 git.OnTrash(f.reader.Forget)
222 f.maint.WithFreshener(f.reader)
223 f.Pushes.WithFreshener(f.reader)
224 reader = f.reader
225 }
226 log.Info().Str("reader", cfg.GitReader).Msg("git reader selected")
227 f.Browse = usecase.NewBrowse(store, reader)
228
229 // Secrets, variables and environments (ADR 0019, CI-3). With a key the
230 // secrets are sealed under it, and the server refuses to start when the
231 // key is not the one they were sealed with; without one they cannot be set
232 // and jobs receive none.
233 var sealer ports.SecretSealer // a nil *secretbox.Box must not become a non-nil interface
234 if cfg.SecretsKey != nil {
235 box, err := secretbox.New(cfg.SecretsKey)
236 if err != nil {
237 return nil, fmt.Errorf("%s: %w", config.EnvSecretsKey, err)
238 }
239 sealer = box
240 }
241 f.Settings, err = usecase.NewCISettings(ctx, store, store.Settings(), sealer, clock)
242 if err != nil {
243 return nil, fmt.Errorf("%s: %w", config.EnvSecretsKey, err)
244 }
245
246 // CI (ADR 0025): the engine reads workflows through the reader, and plans
247 // them in the background; a push only queues its refs.
248 committer := domain.Identity{Name: cfg.CommitterName, Email: cfg.CommitterEmail}
249 ciCfg := usecase.CIConfig{
250 Store: store.CI(), Logs: cilogs.New(cilogs.Dir(cfg.DataDir)), Summaries: cisummaries.New(cisummaries.Dir(cfg.DataDir)), Meta: store, Repos: git, Reader: reader,
251 Git: git, Diff: git, Bus: f.Bus, Clock: clock, Auth: f.Auth,
252 Owner: cfg.Owner, PublicURL: cfg.PublicURL, Retention: cfg.CIRetention,
253 PollTimeout: cfg.CIPollTimeout, Archive: git,
254 Remote: opts.remote(gitremote.New(gitremote.Config{
255 Bin: cfg.Git, Path: path, Home: filepath.Join(cfg.DataDir, "run", "git-home"),
256 Proxy: gitremote.ProxyEnv(os.Getenv),
257 })),
258 Tokens: store.Tokens(),
259 Settings: f.Settings,
260 Workflows: opts.workflows(githubraw.New(githubraw.Config{})),
261 }
262 f.pullRequests(&ciCfg, store, git, committer)
263 if opts.ci != nil {
264 opts.ci(&ciCfg)
265 }
266 f.CI = usecase.NewCI(ctx, ciCfg)
267 f.Pushes.WithWorkflowTrigger(f.CI).WithWorkflowGuard(git)
268 f.Repos.WithCIRunCleaner(f.CI)
269 f.CI.Start()
270 f.EventLog = usecase.NewEvents(ctx, f.Repos, f.Bus, clock, usecase.EventsConfig{
271 Size: opts.eventBuffer, Auth: f.Auth, RevalidateEvery: opts.revalidateEvery,
272 })
273
274 // The instance signing key (ADR 0022): generated on the first start.
275 seed, err := store.SigningSeed(ctx, newSigningSeed, clock.Now().UTC())
276 if err != nil {
277 return nil, err
278 }
279 signer, err := sshsig.New(seed)
280 if err != nil {
281 return nil, err
282 }
283 f.SigningKey = usecase.NewSigningKey(signer, committer)
284 f.MergeRequests = usecase.NewMergeRequests(usecase.MergeRequestsConfig{
285 Meta: store, Store: store.MergeRequests(), Repos: git, Git: git, Signer: signer,
286 Committer: committer, Bus: f.Bus, Clock: clock, Maint: f.maint,
287 CI: f.CI, Checks: f.CI, PullRequests: f.CI,
288 })
289 f.CI.SetPullGate(f.MergeRequests)
290 log.Info().Str("fingerprint", signer.PublicKey().Fingerprint).Msg("commit signing key ready")
291
292 f.Sessions, err = usecase.NewSessions(ctx, store.Sessions(), clock, cfg.PublicURL)
293 if err != nil {
294 return nil, err
295 }
296
297 // Owner passkeys (ADR 0026): built only when the public URL allows them.
298 // When it does, a verifier that cannot be built is fatal.
299 if ok, reason := passkeysAvailable(cfg.PublicURL); ok {
300 verifier, err := webauthn.New(cfg.PublicURL)
301 if err != nil {
302 return nil, fmt.Errorf("passkeys: %w", err)
303 }
304 f.Passkeys = usecase.NewPasskeys(store.Passkeys(), verifier, f.Sessions, clock)
305 log.Info().Str("public_url", cfg.PublicURL).Msg("passkeys enabled")
306 } else {
307 log.Info().Str("reason", reason).Msg("passkeys disabled")
308 }
309
310 // One budget of failed authentications per client, for Git, the API and
311 // the web login.
312 limiter := clientlimit.NewLimiter(nil)
313 // A nil *usecase.Passkeys must not become a non-nil interface: the API
314 // tests for nil to answer failed_precondition and to leave the feature out.
315 var apiPasskeys ports.PasskeyService
316 if f.Passkeys != nil {
317 apiPasskeys = f.Passkeys
318 }
319 f.API = rpc.New(rpc.Config{
320 Auth: f.Auth,
321 Repos: f.Repos,
322 Tokens: f.Tokens,
323 Browse: f.Browse,
324 Events: f.EventLog,
325 MergeRequests: f.MergeRequests,
326 SigningKey: f.SigningKey,
327 Sessions: f.Sessions,
328 Passkeys: apiPasskeys,
329 Runner: f.CI,
330 Runs: f.CI,
331 Secrets: f.Settings,
332 Variables: f.Settings,
333 Environments: f.Settings,
334 Version: serverVersion(),
335 TrustedProxies: cfg.TrustedProxies,
336 Limiter: limiter,
337 })
338
339 gh := githttp.New(githttp.Config{
340 Auth: f.Auth,
341 Repos: f.Repos,
342 Pushes: f.Pushes,
343 Backend: git,
344 Owner: cfg.Owner,
345 Hooks: f.hooks,
346 TrustedProxies: cfg.TrustedProxies,
347 Limiter: limiter,
348 MaxBodyBytes: maxBody,
349 })
350 routes := make([]httpserver.Route, 0, 2*len(githttp.Patterns)+len(githubapi.Patterns)+len(web.Patterns))
351 for _, p := range githttp.Patterns {
352 routes = append(routes, httpserver.Route{Pattern: p, Handler: gh})
353 }
354 // The owner form of the clone URL (github.repository), which GitHub's
355 // actions use. config.Owner follows the rules of repository names, so it is
356 // a literal segment that never overlaps /-/.
357 for _, p := range githttp.OwnerPatterns(cfg.Owner) {
358 routes = append(routes, httpserver.Route{Pattern: p, Handler: gh})
359 }
360 // The REST subset the actions call on github.api_url.
361 api := githubapi.New(githubapi.Config{Auth: f.Auth, API: f.CI, TrustedProxies: cfg.TrustedProxies, Limiter: limiter, MaxArchivesPerClient: opts.archivesPerClient})
362 for _, p := range githubapi.Patterns {
363 routes = append(routes, httpserver.Route{Pattern: p, Handler: api})
364 }
365 warnSessionsNeedHTTPS(ctx, cfg.PublicURL)
366 warnSharedClientKey(ctx, cfg.TrustedProxies, githttp.DefaultMaxPerClient)
367 site, err := newWeb(cfg, f, limiter)
368 if err != nil {
369 return nil, fmt.Errorf("web: %w", err)
370 }
371 routes = append(routes, webRoutes(site)...)
372 f.server = httpserver.New(httpserver.Config{
373 Addr: cfg.Addr,
374 PublicURL: cfg.PublicURL,
375 HealthChecks: []httpserver.Check{store.Check, httpserver.DirWritable(cfg.DataDir)},
376 Routes: routes,
377 API: f.API.Handler(),
378 // Event streams never end by themselves: close them when the
379 // shutdown begins, or it would wait for them until its timeout.
380 OnShutdown: []func(){func() { f.EventLog.Close(context.WithoutCancel(ctx)) }},
381 })
382 return f, nil
383}
384
385// Run listens on the configured address and serves until ctx ends.
386func (f *forge) Run(ctx context.Context) error {
387 ln, err := net.Listen("tcp", f.cfg.Addr)
388 if err != nil {
389 return fmt.Errorf("listen on %s: %w", f.cfg.Addr, err)
390 }
391 return f.Serve(ctx, ln)
392}
393
394// Serve serves HTTP on ln and the hook socket until ctx ends. The hook socket
395// outlives the HTTP server: requests still in flight while it drains are
396// pushes whose hooks need it. A failure of either stops both.
397func (f *forge) Serve(ctx context.Context, ln net.Listener) error {
398 ctx, cancel := context.WithCancel(ctx)
399 defer cancel()
400 hookCtx, stopHooks := context.WithCancel(context.WithoutCancel(ctx))
401 defer stopHooks()
402
403 hookErr := make(chan error, 1)
404 go func() {
405 err := f.hooks.Serve(hookCtx)
406 cancel() // a dead hook socket would reject every push: stop serving
407 hookErr <- err
408 }()
409
410 httpErr := f.server.Serve(ctx, ln)
411 stopHooks()
412 return errors.Join(httpErr, <-hookErr)
413}
414
415// Close releases what newForge opened. It is safe to call twice. Call it after
416// the HTTP server stopped: it stops the event log and the long-lived Git
417// processes of the read side, which requests in flight may still use.
418func (f *forge) Close(ctx context.Context) {
419 log := zerowrap.FromCtx(ctx)
420 if f.hooks != nil {
421 f.hooks.Close(ctx)
422 }
423 if f.CI != nil {
424 f.CI.Close(ctx)
425 }
426 if f.EventLog != nil {
427 f.EventLog.Close(ctx)
428 }
429 if f.reader != nil {
430 f.reader.Close()
431 }
432 if f.git != nil {
433 if err := f.git.Close(ctx); err != nil {
434 log.Error().Err(err).Msg("close git reader")
435 }
436 }
437 if f.maint != nil {
438 f.maint.Close(ctx)
439 }
440 if f.store != nil {
441 if err := f.store.Close(ctx); err != nil {
442 log.Error().Err(err).Msg("close metadata store")
443 }
444 f.store = nil
445 }
446}
447
448// pullRequests gives the CI the merge requests, the refs/pull refs and the
449// identity of the test merge commits (CI-3, D5).
450func (f *forge) pullRequests(cfg *usecase.CIConfig, store *kvstore.Store, git *gitcli.Git, committer domain.Identity) {
451 cfg.MRs, cfg.Pull, cfg.Committer = store.MergeRequests(), git, committer
452 if f.reader != nil {
453 cfg.Fresh = f.reader // a nil *ermitegit.Adapter must not become a non-nil interface
454 }
455}