1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "context"
7 "crypto/ed25519"
8 "crypto/rand"
9 "errors"
10 "fmt"
11 "net"
12 "os"
13 "path/filepath"
14 "time"
15
16 "github.com/bnema/zerowrap"
17
18 "bnema.dev/ermite/internal/adapters/cilogs"
19 "bnema.dev/ermite/internal/adapters/cisummaries"
20 "bnema.dev/ermite/internal/adapters/ermitegit"
21 "bnema.dev/ermite/internal/adapters/eventbus"
22 "bnema.dev/ermite/internal/adapters/gitcli"
23 "bnema.dev/ermite/internal/adapters/githttp"
24 "bnema.dev/ermite/internal/adapters/githubapi"
25 "bnema.dev/ermite/internal/adapters/gitremote"
26 "bnema.dev/ermite/internal/adapters/httpserver"
27 "bnema.dev/ermite/internal/adapters/kvstore"
28 "bnema.dev/ermite/internal/adapters/rpc"
29 "bnema.dev/ermite/internal/adapters/secretbox"
30 "bnema.dev/ermite/internal/adapters/sshsig"
31 "bnema.dev/ermite/internal/adapters/web"
32 "bnema.dev/ermite/internal/adapters/webauthn"
33 "bnema.dev/ermite/internal/clientlimit"
34 "bnema.dev/ermite/internal/config"
35 "bnema.dev/ermite/internal/domain"
36 "bnema.dev/ermite/internal/ports"
37 "bnema.dev/ermite/internal/usecase"
38)
39
40// maxReceiveBytes bounds the body of a push, both at the HTTP handler and in
41// Git (receive.maxInputSize).
42const maxReceiveBytes = githttp.DefaultMaxBodyBytes
43
44// options are the parts of the wiring that tests replace.
45type options struct {
46 // hookExe is the executable the hook scripts exec; empty means this
47 // process's own executable.
48 hookExe string
49 // maintenanceDelay overrides usecase.MaintenanceDelay when positive.
50 maintenanceDelay time.Duration
51 // maxBodyBytes overrides the limit of a push body when positive.
52 maxBodyBytes int64
53 // eventBuffer overrides usecase.DefaultEventBuffer when positive.
54 eventBuffer int
55 // revalidateEvery overrides usecase.DefaultRevalidateEvery when not zero.
56 revalidateEvery time.Duration
57 // archivesPerClient overrides the archives one client downloads at once
58 // when positive.
59 archivesPerClient int
60 // remote replaces the lister of github.com refs (tests never reach the
61 // network).
62 remoteLister ports.GitRemote
63 // ci overrides the settings of the CI engine (tests shorten its
64 // durations).
65 ci func(*usecase.CIConfig)
66}
67
68// remote returns the lister of github.com refs: the test's, else def.
69func (o options) remote(def ports.GitRemote) ports.GitRemote {
70 if o.remoteLister != nil {
71 return o.remoteLister
72 }
73 return def
74}
75
76// forge is the wired server: every adapter and use case, ready to serve.
77type forge struct {
78 cfg config.Config
79 store *kvstore.Store
80 server *httpserver.Server
81 hooks *githttp.HookServer
82 maint *usecase.Maintainer
83 git *gitcli.Git
84 // reader is the ermite-git reader when ERMITE_GIT_READER selects it.
85 reader *ermitegit.Adapter
86
87 // The use cases and the API are exposed for tests and for the phases
88 // that add the web.
89 Repos *usecase.Repos
90 Tokens *usecase.Tokens
91 Auth *usecase.Auth
92 Pushes *usecase.Pushes
93 Browse *usecase.Browse
94 // MergeRequests and SigningKey serve MergeRequestService and
95 // ServerService.GetSigningKey.
96 MergeRequests *usecase.MergeRequests
97 SigningKey *usecase.SigningKey
98 Sessions *usecase.Sessions
99 // Passkeys is nil when passkeys are not available (passkeysAvailable).
100 Passkeys *usecase.Passkeys
101 EventLog *usecase.Events
102 // CI is the CI engine: workflow trigger, planner, runner protocol and the
103 // owner side. It starts with the server and stops with it.
104 CI *usecase.CI
105 // Settings are the secrets, variables and environments of CI.
106 Settings *usecase.CISettings
107 API *rpc.API
108 Bus ports.EventBus
109}
110
111// newSigningSeed makes the seed of a new ed25519 signing key.
112func newSigningSeed() ([]byte, error) {
113 seed := make([]byte, ed25519.SeedSize)
114 if _, err := rand.Read(seed); err != nil {
115 return nil, err
116 }
117 return seed, nil
118}
119
120// serverVersion is the version GetVersion reports: the module version of the
121// build, or "devel" for an untagged one.
122func serverVersion() string {
123 if v := buildVersion(); v != "(devel)" {
124 return v
125 }
126 return "devel"
127}
128
129// newForge opens the metadata store, resets run/, checks git, reconciles the repositories,
130// writes the hook scripts, opens the hook socket and builds the HTTP server.
131// The data directory must be prepared (prepareDataDir). Close it after the
132// server stopped.
133func newForge(ctx context.Context, cfg config.Config, opts options) (_ *forge, err error) {
134 log := zerowrap.FromCtx(ctx)
135 store, err := kvstore.Open(ctx, filepath.Join(cfg.DataDir, "kv"))
136 if err != nil {
137 return nil, err
138 }
139 f := &forge{cfg: cfg, store: store}
140 defer func() {
141 if err != nil {
142 f.Close(context.WithoutCancel(ctx))
143 }
144 }()
145 // The kv store holds the directory lock: only now is run/ ours to reset.
146 // The hook scripts and the socket are written into it afterwards.
147 if err := resetRunDir(cfg.DataDir); err != nil {
148 return nil, err
149 }
150
151 path := os.Getenv("PATH")
152 if path == "" {
153 path = "/usr/local/bin:/usr/bin:/bin"
154 }
155 maxBody := maxReceiveBytes
156 if opts.maxBodyBytes > 0 {
157 maxBody = opts.maxBodyBytes
158 }
159 git := gitcli.New(gitcli.Config{Bin: cfg.Git, DataDir: cfg.DataDir, Path: path, MaxReceiveBytes: maxBody})
160 f.git = git
161 gitVersion, err := git.CheckVersion(ctx)
162 if err != nil {
163 return nil, fmt.Errorf("%s=%q: %w", config.EnvGit, cfg.Git, err)
164 }
165 log.Info().Str("git", gitVersion.String()).Msg("git checked")
166 if _, err := git.HTTPBackend(ctx, false); err != nil {
167 return nil, fmt.Errorf("%s=%q: %w", config.EnvGit, cfg.Git, err)
168 }
169
170 clock := systemClock{}
171 f.Bus = eventbus.New()
172 f.Repos = usecase.NewRepos(store, git, f.Bus, clock)
173 report, err := f.Repos.Reconcile(ctx)
174 if err != nil {
175 return nil, fmt.Errorf("reconcile repositories: %w", err)
176 }
177 log.Info().
178 Int("registered", len(report.Registered)).
179 Int("missing_dirs", len(report.MissingDirs)).
180 Int("invalid_dirs", len(report.InvalidDirs)).
181 Msg("repositories reconciled")
182
183 f.Tokens = usecase.NewTokens(store.Tokens(), clock, store.Sessions())
184 f.Auth = usecase.NewAuth(cfg.AdminTokenHash, store.Tokens(), clock).WithRepoTokens(store.CI())
185 f.maint = usecase.NewMaintainer(ctx, git, opts.maintenanceDelay)
186 f.Pushes = usecase.NewPushes(store, f.Repos, git, f.Bus, clock, f.maint, f.Auth)
187
188 exe := opts.hookExe
189 if exe == "" {
190 if exe, err = os.Executable(); err != nil {
191 return nil, fmt.Errorf("find the ermite executable for the hooks: %w", err)
192 }
193 }
194 if err := githttp.WriteHookScripts(gitcli.HooksDir(cfg.DataDir), exe); err != nil {
195 return nil, err
196 }
197 f.hooks = githttp.NewHookServer(filepath.Join(cfg.DataDir, "run", "hook.sock"), f.Pushes)
198 if err := f.hooks.Listen(ctx); err != nil {
199 return nil, err
200 }
201
202 var reader ports.RepoReader = git
203 if cfg.GitReader == config.GitReaderErmiteGit {
204 f.reader = ermitegit.New(ermitegit.Config{
205 Fallback: git,
206 RepoDir: func(name domain.RepoName) string { return gitcli.RepoDir(cfg.DataDir, name) },
207 })
208 git.OnTrash(f.reader.Forget)
209 f.maint.WithFreshener(f.reader)
210 f.Pushes.WithFreshener(f.reader)
211 reader = f.reader
212 }
213 log.Info().Str("reader", cfg.GitReader).Msg("git reader selected")
214 f.Browse = usecase.NewBrowse(store, reader)
215
216 // Secrets, variables and environments (ADR 0019, CI-3). With a key the
217 // secrets are sealed under it, and the server refuses to start when the
218 // key is not the one they were sealed with; without one they cannot be set
219 // and jobs receive none.
220 var sealer ports.SecretSealer // a nil *secretbox.Box must not become a non-nil interface
221 if cfg.SecretsKey != nil {
222 box, err := secretbox.New(cfg.SecretsKey)
223 if err != nil {
224 return nil, fmt.Errorf("%s: %w", config.EnvSecretsKey, err)
225 }
226 sealer = box
227 }
228 f.Settings, err = usecase.NewCISettings(ctx, store, store.Settings(), sealer, clock)
229 if err != nil {
230 return nil, fmt.Errorf("%s: %w", config.EnvSecretsKey, err)
231 }
232
233 // CI (ADR 0025): the engine reads workflows through the reader, and plans
234 // them in the background; a push only queues its refs.
235 ciCfg := usecase.CIConfig{
236 Store: store.CI(), Logs: cilogs.New(cilogs.Dir(cfg.DataDir)), Summaries: cisummaries.New(cisummaries.Dir(cfg.DataDir)), Meta: store, Repos: git, Reader: reader,
237 Git: git, Diff: git, Bus: f.Bus, Clock: clock, Auth: f.Auth,
238 Owner: cfg.Owner, PublicURL: cfg.PublicURL, Retention: cfg.CIRetention,
239 PollTimeout: cfg.CIPollTimeout, Archive: git,
240 Remote: opts.remote(gitremote.New(gitremote.Config{
241 Bin: cfg.Git, Path: path, Home: filepath.Join(cfg.DataDir, "run", "git-home"),
242 Proxy: gitremote.ProxyEnv(os.Getenv),
243 })),
244 Tokens: store.Tokens(),
245 Settings: f.Settings,
246 }
247 if opts.ci != nil {
248 opts.ci(&ciCfg)
249 }
250 f.CI = usecase.NewCI(ctx, ciCfg)
251 f.Pushes.WithWorkflowTrigger(f.CI)
252 f.Repos.WithCIRunCleaner(f.CI)
253 f.CI.Start()
254 f.EventLog = usecase.NewEvents(ctx, f.Repos, f.Bus, clock, usecase.EventsConfig{
255 Size: opts.eventBuffer, Auth: f.Auth, RevalidateEvery: opts.revalidateEvery,
256 })
257
258 // The instance signing key (ADR 0022): generated on the first start.
259 seed, err := store.SigningSeed(ctx, newSigningSeed, clock.Now().UTC())
260 if err != nil {
261 return nil, err
262 }
263 signer, err := sshsig.New(seed)
264 if err != nil {
265 return nil, err
266 }
267 committer := domain.Identity{Name: cfg.CommitterName, Email: cfg.CommitterEmail}
268 f.SigningKey = usecase.NewSigningKey(signer, committer)
269 f.MergeRequests = usecase.NewMergeRequests(usecase.MergeRequestsConfig{
270 Meta: store, Store: store.MergeRequests(), Repos: git, Git: git, Signer: signer,
271 Committer: committer, Bus: f.Bus, Clock: clock, Maint: f.maint,
272 CI: f.CI, Checks: f.CI,
273 })
274 log.Info().Str("fingerprint", signer.PublicKey().Fingerprint).Msg("commit signing key ready")
275
276 f.Sessions, err = usecase.NewSessions(ctx, store.Sessions(), clock, cfg.PublicURL)
277 if err != nil {
278 return nil, err
279 }
280
281 // Owner passkeys (ADR 0026): built only when the public URL allows them.
282 // When it does, a verifier that cannot be built is fatal.
283 if ok, reason := passkeysAvailable(cfg.PublicURL); ok {
284 verifier, err := webauthn.New(cfg.PublicURL)
285 if err != nil {
286 return nil, fmt.Errorf("passkeys: %w", err)
287 }
288 f.Passkeys = usecase.NewPasskeys(store.Passkeys(), verifier, f.Sessions, clock)
289 log.Info().Str("public_url", cfg.PublicURL).Msg("passkeys enabled")
290 } else {
291 log.Info().Str("reason", reason).Msg("passkeys disabled")
292 }
293
294 // One budget of failed authentications per client, for Git, the API and
295 // the web login.
296 limiter := clientlimit.NewLimiter(nil)
297 // A nil *usecase.Passkeys must not become a non-nil interface: the API
298 // tests for nil to answer failed_precondition and to leave the feature out.
299 var apiPasskeys ports.PasskeyService
300 if f.Passkeys != nil {
301 apiPasskeys = f.Passkeys
302 }
303 f.API = rpc.New(rpc.Config{
304 Auth: f.Auth,
305 Repos: f.Repos,
306 Tokens: f.Tokens,
307 Browse: f.Browse,
308 Events: f.EventLog,
309 MergeRequests: f.MergeRequests,
310 SigningKey: f.SigningKey,
311 Sessions: f.Sessions,
312 Passkeys: apiPasskeys,
313 Runner: f.CI,
314 Runs: f.CI,
315 Secrets: f.Settings,
316 Variables: f.Settings,
317 Environments: f.Settings,
318 Version: serverVersion(),
319 TrustedProxies: cfg.TrustedProxies,
320 Limiter: limiter,
321 })
322
323 gh := githttp.New(githttp.Config{
324 Auth: f.Auth,
325 Repos: f.Repos,
326 Pushes: f.Pushes,
327 Backend: git,
328 Owner: cfg.Owner,
329 Hooks: f.hooks,
330 TrustedProxies: cfg.TrustedProxies,
331 Limiter: limiter,
332 MaxBodyBytes: maxBody,
333 })
334 routes := make([]httpserver.Route, 0, 2*len(githttp.Patterns)+len(githubapi.Patterns)+len(web.Patterns))
335 for _, p := range githttp.Patterns {
336 routes = append(routes, httpserver.Route{Pattern: p, Handler: gh})
337 }
338 // The owner form of the clone URL (github.repository), which GitHub's
339 // actions use. config.Owner follows the rules of repository names, so it is
340 // a literal segment that never overlaps /-/.
341 for _, p := range githttp.OwnerPatterns(cfg.Owner) {
342 routes = append(routes, httpserver.Route{Pattern: p, Handler: gh})
343 }
344 // The REST subset the actions call on github.api_url.
345 api := githubapi.New(githubapi.Config{Auth: f.Auth, API: f.CI, TrustedProxies: cfg.TrustedProxies, Limiter: limiter, MaxArchivesPerClient: opts.archivesPerClient})
346 for _, p := range githubapi.Patterns {
347 routes = append(routes, httpserver.Route{Pattern: p, Handler: api})
348 }
349 warnSessionsNeedHTTPS(ctx, cfg.PublicURL)
350 warnSharedClientKey(ctx, cfg.TrustedProxies, githttp.DefaultMaxPerClient)
351 site, err := newWeb(cfg, f, limiter)
352 if err != nil {
353 return nil, fmt.Errorf("web: %w", err)
354 }
355 routes = append(routes, webRoutes(site)...)
356 f.server = httpserver.New(httpserver.Config{
357 Addr: cfg.Addr,
358 PublicURL: cfg.PublicURL,
359 HealthChecks: []httpserver.Check{store.Check, httpserver.DirWritable(cfg.DataDir)},
360 Routes: routes,
361 API: f.API.Handler(),
362 // Event streams never end by themselves: close them when the
363 // shutdown begins, or it would wait for them until its timeout.
364 OnShutdown: []func(){func() { f.EventLog.Close(context.WithoutCancel(ctx)) }},
365 })
366 return f, nil
367}
368
369// Run listens on the configured address and serves until ctx ends.
370func (f *forge) Run(ctx context.Context) error {
371 ln, err := net.Listen("tcp", f.cfg.Addr)
372 if err != nil {
373 return fmt.Errorf("listen on %s: %w", f.cfg.Addr, err)
374 }
375 return f.Serve(ctx, ln)
376}
377
378// Serve serves HTTP on ln and the hook socket until ctx ends. The hook socket
379// outlives the HTTP server: requests still in flight while it drains are
380// pushes whose hooks need it. A failure of either stops both.
381func (f *forge) Serve(ctx context.Context, ln net.Listener) error {
382 ctx, cancel := context.WithCancel(ctx)
383 defer cancel()
384 hookCtx, stopHooks := context.WithCancel(context.WithoutCancel(ctx))
385 defer stopHooks()
386
387 hookErr := make(chan error, 1)
388 go func() {
389 err := f.hooks.Serve(hookCtx)
390 cancel() // a dead hook socket would reject every push: stop serving
391 hookErr <- err
392 }()
393
394 httpErr := f.server.Serve(ctx, ln)
395 stopHooks()
396 return errors.Join(httpErr, <-hookErr)
397}
398
399// Close releases what newForge opened. It is safe to call twice. Call it after
400// the HTTP server stopped: it stops the event log and the long-lived Git
401// processes of the read side, which requests in flight may still use.
402func (f *forge) Close(ctx context.Context) {
403 log := zerowrap.FromCtx(ctx)
404 if f.hooks != nil {
405 f.hooks.Close(ctx)
406 }
407 if f.CI != nil {
408 f.CI.Close(ctx)
409 }
410 if f.EventLog != nil {
411 f.EventLog.Close(ctx)
412 }
413 if f.reader != nil {
414 f.reader.Close()
415 }
416 if f.git != nil {
417 if err := f.git.Close(ctx); err != nil {
418 log.Error().Err(err).Msg("close git reader")
419 }
420 }
421 if f.maint != nil {
422 f.maint.Close(ctx)
423 }
424 if f.store != nil {
425 if err := f.store.Close(ctx); err != nil {
426 log.Error().Err(err).Msg("close metadata store")
427 }
428 f.store = nil
429 }
430}