1// Package app provides the application initialization and wiring.
2//
3// run.go owns the daemon lifecycle (Run, server start, readiness). Each
4// concern it orders lives in its own file: config.go (load/validate),
5// services.go (service construction), app_wiring.go (apps engine),
6// auth_secrets.go and credentials.go (auth and internal registry
7// credentials), http_edge.go (handlers and middleware), listeners.go
8// (proxy/TLS/registry listeners), reload.go (hot reload), schedulers.go,
9// shutdown.go, and pidfile.go.
10package app
11
12import (
13 "context"
14 "fmt"
15 "net/http"
16 "os"
17 "os/signal"
18 "path/filepath"
19 "syscall"
20 "time"
21
22 "github.com/bnema/zerowrap"
23 zerowrapotel "github.com/bnema/zerowrap/otel"
24 "github.com/spf13/viper"
25
26 "github.com/bnema/gordon/internal/adapters/out/accesslog"
27 "github.com/bnema/gordon/internal/adapters/out/telemetry"
28 "github.com/bnema/gordon/internal/boundaries/out"
29 "github.com/bnema/gordon/internal/usecase/logexport"
30 "github.com/bnema/gordon/pkg/version"
31)
32
33// Run initializes and starts the Gordon application.
34func Run(ctx context.Context, configPath string) error {
35 // Before any secret is loaded: keep same-UID processes out of this
36 // process's /proc entries (environ, mem, fds) and disable core dumps.
37 if err := disableDumpable(); err != nil {
38 return fmt.Errorf("failed to disable process dumping: %w", err)
39 }
40
41 // Load configuration
42 v, cfg, err := initConfig(configPath)
43 if err != nil {
44 return err
45 }
46
47 // Initialize logger
48 log, cleanup, err := initLogger(cfg)
49 if err != nil {
50 return err
51 }
52 if cleanup != nil {
53 defer cleanup()
54 }
55
56 ctx = zerowrap.WithCtx(ctx, log)
57
58 // Initialize OpenTelemetry
59 telProvider, telShutdown, err := telemetry.NewProvider(ctx, cfg.Telemetry, "gordon", version.Version())
60 if err != nil {
61 log.Warn().Err(err).Msg("failed to initialize telemetry, continuing without it")
62 } else {
63 // Use a fresh context for shutdown so a canceled app ctx doesn't prevent flushing.
64 defer func() {
65 shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
66 defer cancel()
67 telShutdown(shutdownCtx)
68 }()
69 if cfg.Telemetry.Enabled && cfg.Telemetry.Endpoint != "" {
70 // Bridge zerowrap logs to OTel if log export is enabled
71 if cfg.Telemetry.Logs && telProvider.LogProvider != nil {
72 otelHook := zerowrapotel.NewHookWithProvider(telProvider.LogProvider, "gordon")
73 log = zerowrap.WithHook(log, otelHook)
74 ctx = zerowrap.WithCtx(ctx, log)
75 }
76 log.Info().Str("endpoint", cfg.Telemetry.Endpoint).Msg("telemetry initialized")
77 }
78 }
79
80 log.Info().Msg("Gordon starting")
81
82 warnDeprecatedConfigKeys(v, log)
83
84 // Create PID file
85 pidFile := createPidFile(log)
86 if pidFile != "" {
87 defer removePidFile(pidFile, log)
88 }
89
90 // Create all services
91 svc, err := createServices(ctx, v, cfg, log)
92 if err != nil {
93 return err
94 }
95 svc.workloadLogs = workloadLogExporter(telProvider)
96
97 // Register event handlers
98 cleanupHandlers, err := registerEventHandlers(ctx, svc)
99 if err != nil {
100 return err
101 }
102 // cleanupHandlers is passed into runServers so it can stop debounce
103 // timers before graceful shutdown, preventing deploys during drain.
104
105 // Set up config hot reload
106 if err := setupConfigHotReload(ctx, svc.configSvc, svc.reloadCoordinator); err != nil {
107 return err
108 }
109
110 // Start event bus
111 if err := svc.eventBus.Start(); err != nil {
112 return log.WrapErr(err, "failed to start event bus")
113 }
114 defer svc.eventBus.Stop()
115
116 // Start servers, wait for listeners to bind, then sync/auto-start containers.
117 return runServers(ctx, v, cfg, svc, svc.reloadCoordinator, cleanupHandlers, log)
118}
119
120// initLogger initializes the zerowrap logger.
121func initLogger(cfg Config) (zerowrap.Logger, func(), error) {
122 logConfig := zerowrap.Config{
123 Level: cfg.Logging.Level,
124 Format: cfg.Logging.Format,
125 }
126
127 // Always enable file logging so admin API can read process logs
128 // (Config flag is respected for backward-compatibility)
129 logPath := cfg.Logging.File.Path
130 if logPath == "" {
131 dataDir := cfg.Server.DataDir
132 if dataDir == "" {
133 dataDir = DefaultDataDir()
134 }
135 logPath = filepath.Join(dataDir, "logs", "gordon.log")
136 }
137
138 log, cleanup, err := zerowrap.NewWithFile(logConfig, zerowrap.FileConfig{
139 Enabled: cfg.Logging.File.Enabled,
140 Path: logPath,
141 MaxSize: cfg.Logging.File.MaxSize,
142 MaxBackups: cfg.Logging.File.MaxBackups,
143 MaxAge: cfg.Logging.File.MaxAge,
144 Compress: true,
145 })
146 if err != nil {
147 return zerowrap.Default(), nil, fmt.Errorf("failed to create logger with file: %w", err)
148 }
149 return log, cleanup, nil
150}
151
152// initAccessLog creates an access log writer when access logging is enabled.
153// Returns nil, nil when disabled — callers must treat nil writer as "disabled".
154func initAccessLog(cfg Config, log zerowrap.Logger) (*accesslog.Writer, error) {
155 if !cfg.Logging.AccessLog.Enabled {
156 return nil, nil
157 }
158
159 filePath := cfg.Logging.AccessLog.FilePath
160 if filePath == "" && cfg.Logging.AccessLog.Output == "file" {
161 dataDir := cfg.Server.DataDir
162 if dataDir == "" {
163 dataDir = DefaultDataDir()
164 }
165 filePath = filepath.Join(dataDir, "logs", "access.log")
166 }
167
168 writer, err := accesslog.New(accesslog.Config{
169 Format: cfg.Logging.AccessLog.Format,
170 Output: cfg.Logging.AccessLog.Output,
171 FilePath: filePath,
172 MaxSize: cfg.Logging.AccessLog.MaxSize,
173 MaxBackups: cfg.Logging.AccessLog.MaxBackups,
174 MaxAge: cfg.Logging.AccessLog.MaxAge,
175 SyslogIdentifier: cfg.Logging.AccessLog.SyslogIdentifier,
176 })
177 if err != nil {
178 return nil, fmt.Errorf("failed to initialize access log: %w", err)
179 }
180
181 log.Info().
182 Str("format", cfg.Logging.AccessLog.Format).
183 Str("output", cfg.Logging.AccessLog.Output).
184 Msg("access log enabled")
185
186 return writer, nil
187}
188
189// runServers starts the HTTP servers and waits for shutdown.
190// Signal handling notes:
191// - SIGINT/SIGTERM: Triggers graceful shutdown via signal.NotifyContext
192// - SIGUSR1: Triggers config reload without restart
193// (SIGUSR2 manual route deploy was removed with the declarative-apps
194// cutover; app deploys are explicit via `gordon apps deploy`.)
195// The deferred signal.Stop calls ensure signal handlers are properly
196// cleaned up before program exit, preventing signal handler leaks.
197func runServers(ctx context.Context, v *viper.Viper, cfg Config, svc *services, reload reloadTrigger, cleanupHandlers func(), log zerowrap.Logger) error {
198 accessWriter, closeAccessLog, err := openAccessLog(svc, cfg, log)
199 if err != nil {
200 return err
201 }
202 defer closeAccessLog()
203 ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
204 defer cancel()
205
206 // Set up SIGUSR1 for reload.
207 // Note: signal.Stop must be called (via defer) to release the channel
208 // and prevent signal handler leaks when the function returns.
209 reloadChan := make(chan os.Signal, 1)
210 signal.Notify(reloadChan, syscall.SIGUSR1)
211 defer signal.Stop(reloadChan)
212
213 errChan := make(chan error, 4)
214
215 registryHandler, httpProxyHandler, httpsProxyHandler := createHTTPHandlers(svc, cfg, log, accessWriter)
216 svc.httpProxyHandler = httpProxyHandler
217 svc.httpsProxyHandler = httpsProxyHandler
218
219 registrySrv, registryReady, internalRegistrySrv, internalRegistryReady, err := startRegistryServers(cfg, registryHandler, errChan, log)
220 if err != nil {
221 return err
222 }
223
224 // closeStarted shuts down any servers that were started before an error occurred,
225 // preventing leaked listeners during partial startup failures.
226 closeStarted := func(servers ...*http.Server) {
227 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
228 defer shutdownCancel()
229 for _, srv := range servers {
230 if srv != nil {
231 if err := srv.Shutdown(shutdownCtx); err != nil {
232 log.Error().Err(err).Msg("failed to shut down server during startup cleanup")
233 }
234 }
235 }
236 }
237
238 proxySrv, proxyReady, tlsSrv, _, err := startProxyServers(cfg, httpProxyHandler, httpsProxyHandler, svc.pkiSvc, svc.publicTLSSvc, svc.trafficManager, log)
239 if err != nil {
240 closeStarted(registrySrv, internalRegistrySrv)
241 return err
242 }
243
244 // Owner-only local administration socket. Started before readiness is
245 // announced so the CLI can reach the daemon as soon as it is usable,
246 // and closed on every exit path. TCP /admin/* registration is unchanged.
247 localAdmin, err := startLocalAdminServer(svc, errChan, log)
248 if err != nil {
249 closeStarted(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
250 shutdownTrafficManagerForStartupCleanup(svc.trafficManager, log)
251 return err
252 }
253
254 // cleanupAfterLocalAdmin releases the local admin socket and traffic
255 // manager in addition to the servers that already bound, so no startup
256 // failure after this point leaks a listening socket or a live manager.
257 cleanupAfterLocalAdmin := func(servers ...*http.Server) {
258 cleanupStartupResources(localAdmin, svc.trafficManager, log, servers...)
259 }
260
261 svc.tlsHTTPEntryPoints = tlsMuxHTTPServerNames(cfg)
262 svc.smartHTTPEntryPoints = smartTCPHTTPServerNames(cfg)
263
264 // Wait for both registry listeners and the HTTP proxy before applying traffic.
265 if err := waitForServerReady(ctx, internalRegistryReady, errChan); err != nil {
266 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
267 return err
268 }
269 if err := waitForCoreProxyReady(ctx, registryReady, proxyReady, errChan); err != nil {
270 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
271 return err
272 }
273 // Apply only the installation graph fail-fast before slower app
274 // readiness checks. Persisted ACTIVE binds are intentionally excluded
275 // until boot reconciliation re-inspects them.
276 if err := applyTrafficRuntimeConfig(ctx, svc.trafficManager, cfg, svc.configSvc, nil); err != nil {
277 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
278 return err
279 }
280
281 // Reconcile ACTIVE and rebuild its host index before announcing the
282 // daemon ready. Applying traffic first publishes an empty app graph and
283 // leaves healthy converged apps returning 404 until a later lifecycle
284 // operation happens to rebuild it.
285 reconcileAppsAtBoot(ctx, svc, log)
286 stopLogExport := startContainerLogExport(ctx, svc, log)
287 defer stopLogExport()
288
289 logEvent := log.Info().
290 Int("proxy_port", cfg.Server.Port).
291 Int("registry_port", cfg.Server.RegistryPort)
292 if cfg.Server.TLSPort != 0 {
293 logEvent = logEvent.Int("tls_port", cfg.Server.TLSPort)
294 }
295 logEvent.Msg("Gordon is running")
296
297 startPublicTLSRuntimeWithWarning(ctx, svc.publicTLSRuntime, log)
298
299 // Seeding lists stored backups (S3 for volumes); never delay startup.
300 go seedBackupMetrics(ctx, svc, log)
301
302 schedulerCleanup, err := startOptionalSchedulers(ctx, cfg, svc, log, v)
303 if err != nil {
304 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
305 return err
306 }
307 if schedulerCleanup != nil {
308 defer schedulerCleanup()
309 }
310
311 waitForShutdown(ctx, errChan, reloadChan, reload, svc.eventBus, log)
312 cleanupHandlers() // Stop debounce timers before draining containers
313 // Stop app administration before tearing down traffic/runtime dependencies,
314 // so no new mutation can begin during shutdown.
315 localAdmin.Close()
316 registrySrvs := []*http.Server{registrySrv, internalRegistrySrv}
317 return gracefulShutdown(registrySrvs, proxySrv, tlsSrv, svc.containerSvc, svc.proxySvc, svc.pkiSvc, svc.publicTLSSvc, svc.trafficManager, svc.appMonitor, svc.appSvcImpl, svc.appState, log)
318}
319
320// workloadLogExporter returns the OTLP workload log exporter, or nil
321// when telemetry log export is disabled. A typed nil must not leak into
322// the out.LogExporter interface.
323func workloadLogExporter(provider *telemetry.Provider) out.LogExporter {
324 if provider == nil || provider.WorkloadLogs == nil {
325 return nil
326 }
327 return provider.WorkloadLogs
328}
329
330// withAccessLogExport also exports access entries over OTLP when
331// workload log export is enabled. next may be nil (no local sink).
332func withAccessLogExport(svc *services, next out.AccessLogWriter) out.AccessLogWriter {
333 if svc.workloadLogs == nil {
334 return next
335 }
336 return logexport.NewAccessLogExporter(svc.appHostIndex, svc.workloadLogs, next)
337}
338
339// startContainerLogExport follows ACTIVE app containers and exports
340// their output when telemetry log export is enabled. The returned stop
341// function cancels every follower and waits for them, so buffered
342// records reach the exporter before telemetry shuts down.
343func startContainerLogExport(ctx context.Context, svc *services, log zerowrap.Logger) func() {
344 if svc.workloadLogs == nil || svc.appState == nil || svc.runtime == nil {
345 return func() {}
346 }
347 ctx, cancel := context.WithCancel(ctx)
348 done := make(chan struct{})
349 collector := logexport.NewCollector(svc.appState, svc.runtime, svc.workloadLogs)
350 go func() {
351 defer close(done)
352 collector.Run(ctx)
353 }()
354 log.Info().Msg("app container log export enabled")
355 return func() {
356 cancel()
357 <-done
358 }
359}
360
361func waitForCoreProxyReady(ctx context.Context, registryReady <-chan struct{}, proxyReady <-chan struct{}, errChan <-chan error) error {
362 if err := waitForServerReady(ctx, registryReady, errChan); err != nil {
363 return err
364 }
365 return waitForServerReady(ctx, proxyReady, errChan)
366}
367
368func waitForServerReady(ctx context.Context, ready <-chan struct{}, errChan <-chan error) error {
369 if ready == nil {
370 return nil
371 }
372 select {
373 case <-ready:
374 return nil
375 case err := <-errChan:
376 return err
377 case <-ctx.Done():
378 return ctx.Err()
379 }
380}
381
382// openAccessLog initializes the access log writer and returns the writer to use
383// (nil when disabled) with its close function. Kept out of runServers to keep
384// its cyclomatic complexity within the project limit of 15.
385func openAccessLog(svc *services, cfg Config, log zerowrap.Logger) (out.AccessLogWriter, func(), error) {
386 accessWriterConcrete, err := initAccessLog(cfg, log)
387 if err != nil {
388 return nil, func() {}, err
389 }
390 closeFn := func() {}
391 // Convert to interface only when non-nil to avoid the Go nil-interface pitfall
392 // where a typed nil pointer becomes a non-nil interface value.
393 var accessWriter out.AccessLogWriter
394 if accessWriterConcrete != nil {
395 accessWriter = accessWriterConcrete
396 closeFn = func() { accessWriterConcrete.Close() }
397 }
398 return withAccessLogExport(svc, accessWriter), closeFn, nil
399}