1// Package app provides the application initialization and wiring.
2package app
3
4import (
5 "context"
6 "crypto/rand"
7 "crypto/tls"
8 "crypto/x509"
9 "encoding/hex"
10 "encoding/json"
11 "errors"
12 "fmt"
13 "net"
14 "net/http"
15 "os"
16 "os/signal"
17 "path/filepath"
18 "strconv"
19 "strings"
20 "sync"
21 "syscall"
22 "time"
23
24 "github.com/bnema/zerowrap"
25 zerowrapotel "github.com/bnema/zerowrap/otel"
26 "github.com/spf13/viper"
27 "golang.org/x/sys/unix"
28
29 // Adapters - Output
30 "github.com/bnema/gordon/internal/adapters/out/accesslog"
31 acmelego "github.com/bnema/gordon/internal/adapters/out/acmelego"
32 acmestore "github.com/bnema/gordon/internal/adapters/out/acmestore"
33 "github.com/bnema/gordon/internal/adapters/out/docker"
34 "github.com/bnema/gordon/internal/adapters/out/eventbus"
35 "github.com/bnema/gordon/internal/adapters/out/filesystem"
36 "github.com/bnema/gordon/internal/adapters/out/httpprober"
37 "github.com/bnema/gordon/internal/adapters/out/logwriter"
38 pkiadapter "github.com/bnema/gordon/internal/adapters/out/pki"
39 "github.com/bnema/gordon/internal/adapters/out/ratelimit"
40 s3storage "github.com/bnema/gordon/internal/adapters/out/s3"
41 "github.com/bnema/gordon/internal/adapters/out/secrets"
42 "github.com/bnema/gordon/internal/adapters/out/telemetry"
43 "github.com/bnema/gordon/internal/adapters/out/tokenstore"
44
45 // OTel
46 "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
47
48 // Adapters - Input
49 "github.com/bnema/gordon/internal/adapters/dto"
50 acmehttp "github.com/bnema/gordon/internal/adapters/in/http/acme"
51 "github.com/bnema/gordon/internal/adapters/in/http/admin"
52 authhandler "github.com/bnema/gordon/internal/adapters/in/http/auth"
53 "github.com/bnema/gordon/internal/adapters/in/http/httphelper"
54 "github.com/bnema/gordon/internal/adapters/in/http/middleware"
55 "github.com/bnema/gordon/internal/adapters/in/http/onboarding"
56 proxyadapter "github.com/bnema/gordon/internal/adapters/in/http/proxy"
57 "github.com/bnema/gordon/internal/adapters/in/http/registry"
58 trafficadapter "github.com/bnema/gordon/internal/adapters/in/traffic"
59
60 // Boundaries
61 "github.com/bnema/gordon/internal/boundaries/in"
62 "github.com/bnema/gordon/internal/boundaries/out"
63
64 // Domain
65 "github.com/bnema/gordon/internal/domain"
66
67 // Packages
68 "github.com/bnema/gordon/pkg/version"
69
70 // Use cases
71 "github.com/bnema/gordon/internal/adapters/out/appsecrets"
72 "github.com/bnema/gordon/internal/adapters/out/appstate"
73 "github.com/bnema/gordon/internal/adapters/out/imageref"
74 "github.com/bnema/gordon/internal/usecase/apps"
75 "github.com/bnema/gordon/internal/usecase/apptraffic"
76 "github.com/bnema/gordon/internal/usecase/auth"
77 "github.com/bnema/gordon/internal/usecase/backup"
78 "github.com/bnema/gordon/internal/usecase/config"
79 "github.com/bnema/gordon/internal/usecase/container"
80 cronSvc "github.com/bnema/gordon/internal/usecase/cron"
81 "github.com/bnema/gordon/internal/usecase/deployment"
82 "github.com/bnema/gordon/internal/usecase/health"
83 "github.com/bnema/gordon/internal/usecase/images"
84 "github.com/bnema/gordon/internal/usecase/logexport"
85 "github.com/bnema/gordon/internal/usecase/logs"
86 pkiusecase "github.com/bnema/gordon/internal/usecase/pki"
87 "github.com/bnema/gordon/internal/usecase/proxy"
88 "github.com/bnema/gordon/internal/usecase/publictls"
89 registrySvc "github.com/bnema/gordon/internal/usecase/registry"
90 "github.com/bnema/gordon/internal/usecase/registrystate"
91 servicecfg "github.com/bnema/gordon/internal/usecase/services"
92 "github.com/bnema/gordon/internal/usecase/traffic"
93 volumesSvc "github.com/bnema/gordon/internal/usecase/volumes"
94
95 // Pkg
96 "github.com/bnema/gordon/pkg/bytesize"
97 "github.com/bnema/gordon/pkg/duration"
98)
99
100// Config holds the application configuration.
101type Config struct {
102 Server struct {
103 Port int `mapstructure:"port"`
104 RegistryPort int `mapstructure:"registry_port"`
105 GordonDomain string `mapstructure:"gordon_domain"`
106 RegistryDomain string `mapstructure:"registry_domain"`
107 LegacyRegistryDomains []string `mapstructure:"legacy_registry_domains"`
108 TLSPort int `mapstructure:"tls_port"`
109 TLSCertFile string `mapstructure:"tls_cert_file"`
110 TLSKeyFile string `mapstructure:"tls_key_file"`
111 ForceHTTPSRedirect bool `mapstructure:"force_https_redirect"`
112 DataDir string `mapstructure:"data_dir"`
113 MaxProxyBodySize string `mapstructure:"max_proxy_body_size"` // e.g., "512MB", "1GB"
114 MaxBlobChunkSize string `mapstructure:"max_blob_chunk_size"` // e.g., "512MB", "1GB"
115 MaxBlobSize string `mapstructure:"max_blob_size"` // e.g., "1GB", "2GB"
116 MaxProxyResponseSize string `mapstructure:"max_proxy_response_size"` // e.g., "1GB", "0" for no limit
117 MaxConcurrentConns int `mapstructure:"max_concurrent_connections"`
118 RegistryAllowedIPs []string `mapstructure:"registry_allowed_ips"`
119 ProxyAllowedIPs []string `mapstructure:"proxy_allowed_ips"`
120 RegistryListenAddr string `mapstructure:"registry_listen_address"`
121 } `mapstructure:"server"`
122
123 Apps struct {
124 // RevisionRetention bounds unreferenced-revision GC per app.
125 // Zero or negative restores the compiled default (8).
126 RevisionRetention int `mapstructure:"revision_retention"`
127 } `mapstructure:"apps"`
128
129 // AppMounts declares administrative bind policies keyed by stable mount
130 // name, referenced by name from an app manifest's [[services.<name>.bind]].
131 AppMounts map[string]AppMountPolicy `mapstructure:"app_mounts"`
132
133 // AppDevices declares administrative device grants keyed by stable
134 // logical device name, referenced by name from an app manifest's
135 // `devices` list.
136 AppDevices map[string]AppDevicePolicy `mapstructure:"app_devices"`
137
138 Logging struct {
139 Level string `mapstructure:"level"`
140 Format string `mapstructure:"format"`
141 File struct {
142 Enabled bool `mapstructure:"enabled"`
143 Path string `mapstructure:"path"`
144 MaxSize int `mapstructure:"max_size"`
145 MaxBackups int `mapstructure:"max_backups"`
146 MaxAge int `mapstructure:"max_age"`
147 } `mapstructure:"file"`
148 ContainerLogs struct {
149 Enabled bool `mapstructure:"enabled"`
150 Dir string `mapstructure:"dir"`
151 MaxSize int `mapstructure:"max_size"`
152 MaxBackups int `mapstructure:"max_backups"`
153 MaxAge int `mapstructure:"max_age"`
154 } `mapstructure:"container_logs"`
155 AccessLog struct {
156 Enabled bool `mapstructure:"enabled"`
157 Format string `mapstructure:"format"`
158 Output string `mapstructure:"output"`
159 FilePath string `mapstructure:"file_path"`
160 MaxSize int `mapstructure:"max_size"`
161 MaxBackups int `mapstructure:"max_backups"`
162 MaxAge int `mapstructure:"max_age"`
163 ExcludeHealthChecks bool `mapstructure:"exclude_health_checks"`
164 SyslogIdentifier string `mapstructure:"syslog_identifier"`
165 } `mapstructure:"access_log"`
166 } `mapstructure:"logging"`
167
168 Volumes struct {
169 AutoCreate bool `mapstructure:"auto_create"`
170 Prefix string `mapstructure:"prefix"`
171 Preserve bool `mapstructure:"preserve"`
172 } `mapstructure:"volumes"`
173
174 Auth struct {
175 Enabled bool `mapstructure:"enabled"`
176 Type string `mapstructure:"type"` // only "token" is supported
177 SecretsBackend string `mapstructure:"secrets_backend"` // "pass", "sops", or "unsafe"
178 Username string `mapstructure:"username"`
179 TokenSecret string `mapstructure:"token_secret"` // path in secrets backend
180 TokenExpiry string `mapstructure:"token_expiry"` // e.g., "720h", "30d"
181 AccessTokenTTL string `mapstructure:"access_token_ttl"` // e.g., "15m", "30m" (default: 15m)
182 } `mapstructure:"auth"`
183
184 API struct {
185 RateLimit struct {
186 Enabled bool `mapstructure:"enabled"`
187 GlobalRPS float64 `mapstructure:"global_rps"`
188 PerIPRPS float64 `mapstructure:"per_ip_rps"`
189 Burst int `mapstructure:"burst"`
190 TrustedProxies []string `mapstructure:"trusted_proxies"`
191 } `mapstructure:"rate_limit"`
192 } `mapstructure:"api"`
193
194 EntryPoints map[string]traffic.EntryPointConfig `mapstructure:"entrypoints"`
195 Traffic traffic.Config `mapstructure:"traffic"`
196 NetworkServices []traffic.NetworkServiceConfig `mapstructure:"network_services"`
197 Services []servicecfg.Config `mapstructure:"services"`
198
199 Backups struct {
200 // Legacy database backup keys. Prefer backups.databases.* for new configs.
201 Enabled bool `mapstructure:"enabled"`
202 Schedule string `mapstructure:"schedule"`
203 StorageDir string `mapstructure:"storage_dir"`
204 Retention struct {
205 Hourly int `mapstructure:"hourly"`
206 Daily int `mapstructure:"daily"`
207 Weekly int `mapstructure:"weekly"`
208 Monthly int `mapstructure:"monthly"`
209 } `mapstructure:"retention"`
210 Databases struct {
211 Enabled bool `mapstructure:"enabled"`
212 Schedule string `mapstructure:"schedule"`
213 StorageDir string `mapstructure:"storage_dir"`
214 Retention struct {
215 Hourly int `mapstructure:"hourly"`
216 Daily int `mapstructure:"daily"`
217 Weekly int `mapstructure:"weekly"`
218 Monthly int `mapstructure:"monthly"`
219 } `mapstructure:"retention"`
220 } `mapstructure:"databases"`
221 Volumes struct {
222 Enabled bool `mapstructure:"enabled"`
223 Interval string `mapstructure:"interval"`
224 Compression string `mapstructure:"compression"`
225 Timeout string `mapstructure:"timeout"`
226 MaxConcurrency int `mapstructure:"max_concurrency"`
227 HelperImage string `mapstructure:"helper_image"`
228 S3 struct {
229 Bucket string `mapstructure:"bucket"`
230 Region string `mapstructure:"region"`
231 Prefix string `mapstructure:"prefix"`
232 Endpoint string `mapstructure:"endpoint"`
233 PathStyle bool `mapstructure:"path_style"`
234 SSEAlgorithm string `mapstructure:"sse_algorithm"`
235 SSEKMSKeyID string `mapstructure:"sse_kms_key_id"`
236 } `mapstructure:"s3"`
237 Retention struct {
238 Keep int `mapstructure:"keep"`
239 } `mapstructure:"retention"`
240 } `mapstructure:"volumes"`
241 } `mapstructure:"backups"`
242
243 Images struct {
244 AllowedRegistries []string `mapstructure:"allowed_registries"`
245 RequireDigest bool `mapstructure:"require_digest"`
246 Prune struct {
247 Enabled bool `mapstructure:"enabled"`
248 Schedule string `mapstructure:"schedule"`
249 KeepLast int `mapstructure:"keep_last"`
250 } `mapstructure:"prune"`
251 } `mapstructure:"images"`
252
253 Containers struct {
254 MemoryLimit string `mapstructure:"memory_limit"` // e.g., "512MB", "1GB"
255 CPULimit float64 `mapstructure:"cpu_limit"` // CPU cores, e.g., 1.0 = 1 core
256 PidsLimit int64 `mapstructure:"pids_limit"` // e.g., 512
257 SecurityProfile string `mapstructure:"security_profile"` // compat or strict
258 } `mapstructure:"containers"`
259
260 NetworkIsolation struct {
261 Enabled bool `mapstructure:"enabled"`
262 Prefix string `mapstructure:"network_prefix"`
263 Internal bool `mapstructure:"internal"`
264 } `mapstructure:"network_isolation"`
265
266 Telemetry telemetry.Config `mapstructure:"telemetry"`
267
268 TLS struct {
269 ACME struct {
270 Enabled bool `mapstructure:"enabled"`
271 Email string `mapstructure:"email"`
272 Challenge string `mapstructure:"challenge"`
273 ObtainBatchSize int `mapstructure:"obtain_batch_size"`
274 } `mapstructure:"acme"`
275 } `mapstructure:"tls"`
276
277 DNS struct {
278 Resolvers []string `mapstructure:"resolvers"`
279 PropagationTimeout string `mapstructure:"propagation_timeout"`
280 PollingInterval string `mapstructure:"polling_interval"`
281 } `mapstructure:"dns"`
282}
283
284// services holds all the services used by the application.
285type services struct {
286 runtime *docker.Runtime
287 eventBus *eventbus.InMemory
288 metrics *telemetry.Metrics
289 blobStorage *filesystem.BlobStorage
290 manifestStorage *filesystem.ManifestStorage
291 backupStorage *filesystem.BackupStorage
292 volumeBackupStore out.VolumeBackupStorage
293 volumeBackupCfg domain.VolumeBackupConfig
294 logWriter *logwriter.LogWriter
295 tokenStore out.TokenStore
296 configSvc *config.Service
297 containerSvc *container.Service
298 backupSvc *backup.Service
299 volumeBackupSvc *backup.VolumeService
300 registrySvc *registrySvc.Service
301 healthSvc *health.Service
302 logSvc *logs.Service
303 imageSvc *images.Service
304 volumeSvc *volumesSvc.Service
305 proxySvc *proxy.Service
306 serviceSecretProvider out.SecretProvider
307 authSvc *auth.Service
308 authHandler *authhandler.Handler
309 adminHandler *admin.Handler
310 httpProxyHandler http.Handler
311 httpsProxyHandler http.Handler
312 internalRegUser string
313 internalRegPass string
314 maxBlobChunkSize int64
315 maxBlobSize int64
316 caAdapter *pkiadapter.CA
317 pkiSvc *pkiusecase.Service
318 appState out.AppState
319 // gcBarrier serializes resource acquisition and its durable
320 // protection publication against destructive prune.
321 gcBarrier out.GCBarrier
322 appDeploySvc *deployment.Service
323 appSvc in.AppService
324 appSvcImpl *apps.AppServiceImpl
325 appActivator *apptraffic.Activator
326 appHostIndex *apptraffic.HostIndex
327 appTrafficPublisher *appTrafficPublisher
328 appMonitor *appMonitor
329 reloadCoordinator *reloadCoordinator
330 publicTLSSvc in.PublicTLSService
331 publicTLSRuntime publicTLSRuntime
332 trafficManager *trafficadapter.Manager
333 tlsHTTPEntryPoints map[string]struct{}
334 smartHTTPEntryPoints map[string]struct{}
335 registryHandler interface {
336 UpdateBlobLimits(maxBlobChunkSize, maxBlobSize int64)
337 }
338 // workloadLogs exports proxy access and app container logs over
339 // OTLP. Nil when telemetry log export is disabled.
340 workloadLogs out.LogExporter
341}
342
343// Run initializes and starts the Gordon application.
344func Run(ctx context.Context, configPath string) error {
345 // Load configuration
346 v, cfg, err := initConfig(configPath)
347 if err != nil {
348 return err
349 }
350
351 // Initialize logger
352 log, cleanup, err := initLogger(cfg)
353 if err != nil {
354 return err
355 }
356 if cleanup != nil {
357 defer cleanup()
358 }
359
360 ctx = zerowrap.WithCtx(ctx, log)
361
362 // Initialize OpenTelemetry
363 telProvider, telShutdown, err := telemetry.NewProvider(ctx, cfg.Telemetry, "gordon", version.Version())
364 if err != nil {
365 log.Warn().Err(err).Msg("failed to initialize telemetry, continuing without it")
366 } else {
367 // Use a fresh context for shutdown so a canceled app ctx doesn't prevent flushing.
368 defer func() {
369 shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
370 defer cancel()
371 telShutdown(shutdownCtx)
372 }()
373 if cfg.Telemetry.Enabled && cfg.Telemetry.Endpoint != "" {
374 // Bridge zerowrap logs to OTel if log export is enabled
375 if cfg.Telemetry.Logs && telProvider.LogProvider != nil {
376 otelHook := zerowrapotel.NewHookWithProvider(telProvider.LogProvider, "gordon")
377 log = zerowrap.WithHook(log, otelHook)
378 ctx = zerowrap.WithCtx(ctx, log)
379 }
380 log.Info().Str("endpoint", cfg.Telemetry.Endpoint).Msg("telemetry initialized")
381 }
382 }
383
384 log.Info().Msg("Gordon starting")
385
386 warnDeprecatedConfigKeys(v, log)
387
388 // Create PID file
389 pidFile := createPidFile(log)
390 if pidFile != "" {
391 defer removePidFile(pidFile, log)
392 }
393
394 // Create all services
395 svc, err := createServices(ctx, v, cfg, log)
396 if err != nil {
397 return err
398 }
399 svc.workloadLogs = workloadLogExporter(telProvider)
400
401 // Register event handlers
402 cleanupHandlers, err := registerEventHandlers(ctx, svc)
403 if err != nil {
404 return err
405 }
406 // cleanupHandlers is passed into runServers so it can stop debounce
407 // timers before graceful shutdown, preventing deploys during drain.
408
409 // Set up config hot reload
410 if err := setupConfigHotReload(ctx, svc.configSvc, svc.reloadCoordinator); err != nil {
411 return err
412 }
413
414 // Start event bus
415 if err := svc.eventBus.Start(); err != nil {
416 return log.WrapErr(err, "failed to start event bus")
417 }
418 defer svc.eventBus.Stop()
419
420 // Start servers, wait for listeners to bind, then sync/auto-start containers.
421 return runServers(ctx, v, cfg, svc, svc.reloadCoordinator, cleanupHandlers, log)
422}
423
424func warnDeprecatedConfigKeys(v *viper.Viper, log zerowrap.Logger) {
425 for _, key := range []string{"server.tls_enabled", "server.force_hsts"} {
426 if v.IsSet(key) {
427 log.Warn().Str("key", key).Msg("deprecated config key — Gordon now uses an internal CA with automatic TLS; remove this from your config")
428 }
429 }
430}
431
432// initConfig loads configuration from file.
433func initConfig(configPath string) (*viper.Viper, Config, error) {
434 v := viper.New()
435 if err := loadConfig(v, configPath); err != nil {
436 return nil, Config{}, fmt.Errorf("failed to load config: %w", err)
437 }
438
439 var cfg Config
440 if err := v.Unmarshal(&cfg); err != nil {
441 return nil, Config{}, fmt.Errorf("failed to unmarshal config: %w", err)
442 }
443 if err := validateEntrypointMigration(v, cfg); err != nil {
444 return nil, Config{}, err
445 }
446 if err := validateRetiredAppConfig(v); err != nil {
447 return nil, Config{}, err
448 }
449 if _, err := buildAppMountPolicies(cfg); err != nil {
450 return nil, Config{}, err
451 }
452 if err := validateDeviceConfig(v, cfg); err != nil {
453 return nil, Config{}, err
454 }
455
456 return v, cfg, nil
457}
458
459// retiredAppConfigKeys are pre-v3 application keys removed with the
460// declarative-apps cutover. Presence fails boot/reload closed with a
461// config-retired diagnostic naming the fix — never a silent migration.
462// Keys that remain live installation-level settings are deliberately
463// absent: top-level "services" still declares standalone L4 workloads
464// (see config.services), so it must never be rejected here.
465var retiredAppConfigKeys = []struct {
466 key string
467 hint string
468}{
469 {"routes", "declare [[services.<name>.http]] in an app file, then 'gordon apps apply'"},
470 {"attachments", "declare [services.<name>] + volumes; attachments are removed"},
471 {"network_groups", "declare [[network.shared]] with ownership verification"},
472 {"service_routes", "declare [[services.<name>.http]] instead"},
473 {"auto", "feature removed; declare explicit interfaces"},
474 {"auto_route", "feature removed; declare explicit interfaces"},
475 {"auto_route_allowed_domains", "feature removed; declare explicit interfaces"},
476 {"previews", "staging is an ordinary app file"},
477 {"env", "remove the [env] section; declare [env] and [services.<name>.secrets] in app files"},
478}
479
480// validateRetiredAppConfig rejects obsolete application configuration
481// BEFORE any app/runtime mutation, at startup and reload. InConfig
482// (not IsSet) targets explicit file keys only — installation defaults
483// registered via SetDefault must not trip the rejection.
484func validateRetiredAppConfig(v *viper.Viper) error {
485 diagnostics := make([]string, 0, len(retiredAppConfigKeys))
486 for _, retired := range retiredAppConfigKeys {
487 if v.InConfig(retired.key) {
488 diagnostics = append(diagnostics, fmt.Sprintf("key %q was removed in v3; %s", retired.key, retired.hint))
489 }
490 }
491 if len(diagnostics) > 0 {
492 return fmt.Errorf("config-retired: %s", strings.Join(diagnostics, "; "))
493 }
494 return nil
495}
496
497func validateEntrypointMigration(v *viper.Viper, cfg Config) error {
498 if len(cfg.EntryPoints) > 0 {
499 return nil
500 }
501
502 legacyKeys := make([]string, 0, 2)
503 for _, key := range []string{"server.port", "server.tls_port"} {
504 if v.IsSet(key) {
505 legacyKeys = append(legacyKeys, key)
506 }
507 }
508 if len(legacyKeys) == 0 {
509 return nil
510 }
511
512 return fmt.Errorf("legacy %s configuration requires at least one [entrypoints] entry; see docs/upgrading.md", strings.Join(legacyKeys, " or "))
513}
514
515// initLogger initializes the zerowrap logger.
516func initLogger(cfg Config) (zerowrap.Logger, func(), error) {
517 logConfig := zerowrap.Config{
518 Level: cfg.Logging.Level,
519 Format: cfg.Logging.Format,
520 }
521
522 // Always enable file logging so admin API can read process logs
523 // (Config flag is respected for backward-compatibility)
524 logPath := cfg.Logging.File.Path
525 if logPath == "" {
526 dataDir := cfg.Server.DataDir
527 if dataDir == "" {
528 dataDir = DefaultDataDir()
529 }
530 logPath = filepath.Join(dataDir, "logs", "gordon.log")
531 }
532
533 log, cleanup, err := zerowrap.NewWithFile(logConfig, zerowrap.FileConfig{
534 Enabled: cfg.Logging.File.Enabled,
535 Path: logPath,
536 MaxSize: cfg.Logging.File.MaxSize,
537 MaxBackups: cfg.Logging.File.MaxBackups,
538 MaxAge: cfg.Logging.File.MaxAge,
539 Compress: true,
540 })
541 if err != nil {
542 return zerowrap.Default(), nil, fmt.Errorf("failed to create logger with file: %w", err)
543 }
544 return log, cleanup, nil
545}
546
547// initAccessLog creates an access log writer when access logging is enabled.
548// Returns nil, nil when disabled — callers must treat nil writer as "disabled".
549func initAccessLog(cfg Config, log zerowrap.Logger) (*accesslog.Writer, error) {
550 if !cfg.Logging.AccessLog.Enabled {
551 return nil, nil
552 }
553
554 filePath := cfg.Logging.AccessLog.FilePath
555 if filePath == "" && cfg.Logging.AccessLog.Output == "file" {
556 dataDir := cfg.Server.DataDir
557 if dataDir == "" {
558 dataDir = DefaultDataDir()
559 }
560 filePath = filepath.Join(dataDir, "logs", "access.log")
561 }
562
563 writer, err := accesslog.New(accesslog.Config{
564 Format: cfg.Logging.AccessLog.Format,
565 Output: cfg.Logging.AccessLog.Output,
566 FilePath: filePath,
567 MaxSize: cfg.Logging.AccessLog.MaxSize,
568 MaxBackups: cfg.Logging.AccessLog.MaxBackups,
569 MaxAge: cfg.Logging.AccessLog.MaxAge,
570 SyslogIdentifier: cfg.Logging.AccessLog.SyslogIdentifier,
571 })
572 if err != nil {
573 return nil, fmt.Errorf("failed to initialize access log: %w", err)
574 }
575
576 log.Info().
577 Str("format", cfg.Logging.AccessLog.Format).
578 Str("output", cfg.Logging.AccessLog.Output).
579 Msg("access log enabled")
580
581 return writer, nil
582}
583
584// serviceInit holds the shared context for service initialization helpers.
585type serviceInit struct {
586 ctx context.Context
587 v *viper.Viper
588 cfg Config
589 log zerowrap.Logger
590 svc *services
591}
592
593// createServices creates all the application services for server runtime.
594// Public ACME reconciliation is started later, after the HTTP listener is bound.
595func createServices(ctx context.Context, v *viper.Viper, cfg Config, log zerowrap.Logger) (_ *services, retErr error) {
596 return createServicesWithOptions(ctx, v, cfg, log)
597}
598
599// createServicesWithOptions creates all the application services.
600// ACME Reconcile and renewal loop are started later from runServers,
601// after HTTP listeners are bound.
602func createServicesWithOptions(ctx context.Context, v *viper.Viper, cfg Config, log zerowrap.Logger) (_ *services, retErr error) {
603 si := &serviceInit{
604 ctx: ctx,
605 v: v,
606 cfg: cfg,
607 log: log,
608 svc: &services{},
609 }
610 defer func() {
611 if retErr != nil {
612 if si.svc.pkiSvc != nil {
613 si.svc.pkiSvc.Stop()
614 }
615 if si.svc.publicTLSSvc != nil {
616 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
617 defer cancel()
618 if err := si.svc.publicTLSSvc.Stop(ctx); err != nil {
619 si.log.Warn().Err(err).Msg("failed to stop public TLS service during createServices cleanup")
620 }
621 }
622 }
623 }()
624 var err error
625
626 // Create output adapters
627 runtimeSocket := resolveRuntimeConfig(v.GetString("server.runtime"))
628 if si.svc.runtime, si.svc.eventBus, err = createOutputAdapters(ctx, log, runtimeSocket); err != nil {
629 return nil, err
630 }
631
632 // Create storage
633 if si.svc.blobStorage, si.svc.manifestStorage, err = createStorage(cfg, log); err != nil {
634 return nil, err
635 }
636
637 // Create log writer
638 if si.svc.logWriter, err = createLogWriter(cfg, log); err != nil {
639 return nil, err
640 }
641
642 // Create auth service (if enabled)
643 if si.svc.tokenStore, si.svc.authSvc, err = createAuthService(ctx, cfg, log); err != nil {
644 return nil, err
645 }
646
647 if err := setupInternalRegistryAuth(si.svc, log); err != nil {
648 return nil, err
649 }
650
651 // Create config service
652 si.svc.configSvc = config.NewService(v, si.svc.eventBus)
653 if err := si.svc.configSvc.Load(ctx); err != nil {
654 return nil, log.WrapErr(err, "failed to load configuration")
655 }
656
657 if err := si.initPKI(); err != nil {
658 return nil, err
659 }
660
661 if err := si.initSecrets(); err != nil {
662 return nil, err
663 }
664
665 if err := si.initPublicTLS(); err != nil {
666 return nil, err
667 }
668
669 if err := si.initRuntimeProxyAndTraffic(); err != nil {
670 return nil, err
671 }
672
673 si.svc.reloadCoordinator = newReloadCoordinator(v, si.svc.configSvc, si.svc.proxySvc, nil, si.svc.eventBus, si.svc.publicTLSSvc, log)
674 si.registerReloadCoordinatorHooks()
675
676 si.initHandlers()
677
678 return si.svc, nil
679}
680
681// initPKI initialises the internal CA and PKI service when TLS is enabled.
682func (si *serviceInit) initPKI() error {
683 if !hasTLSCapableEntrypoint(si.cfg) {
684 si.log.Info().Msg("internal CA disabled (no TLS-capable entrypoint configured)")
685 return nil
686 }
687
688 if (si.cfg.Server.TLSCertFile == "") != (si.cfg.Server.TLSKeyFile == "") {
689 return fmt.Errorf("both tls_cert_file and tls_key_file must be set, or neither")
690 }
691
692 caAdapter, err := pkiadapter.NewCA(resolveDataDir(si.cfg.Server.DataDir), si.log)
693 if err != nil {
694 return si.log.WrapErr(err, "failed to initialize internal CA")
695 }
696 si.svc.caAdapter = caAdapter
697 si.svc.pkiSvc = pkiusecase.NewService(si.ctx, caAdapter, si.appRoutes(), []string{si.cfg.Server.GordonDomain}, si.log)
698 return nil
699}
700
701// initPublicTLS initializes the public ACME TLS service if enabled.
702func (si *serviceInit) initPublicTLS() error {
703 if !si.cfg.TLS.ACME.Enabled {
704 return nil
705 }
706
707 if err := validatePublicTLSReadiness(si.cfg); err != nil {
708 return err
709 }
710
711 ctx := si.ctx
712 log := si.log
713
714 dnsCfg, err := buildDNSConfig(si.cfg)
715 if err != nil {
716 return log.WrapErr(err, "invalid DNS configuration")
717 }
718
719 publicTLSCfg := publictls.Config{
720 Enabled: si.cfg.TLS.ACME.Enabled,
721 Email: si.cfg.TLS.ACME.Email,
722 Challenge: si.cfg.TLS.ACME.Challenge,
723 HTTPPort: effectiveHTTP01Port(si.cfg),
724 TLSPort: effectivePublicTLSPort(si.cfg),
725 DataDir: resolveDataDir(si.cfg.Server.DataDir),
726 ObtainBatchSize: si.cfg.TLS.ACME.ObtainBatchSize,
727 DNS: dnsCfg,
728 }
729
730 tokenResolver := secrets.NewPublicTLSResolver(secrets.PublicTLSResolverConfig{})
731
732 effective, err := publictls.ResolveEffectiveChallenge(ctx, publicTLSCfg, tokenResolver)
733 if err != nil {
734 return log.WrapErr(err, "resolve ACME challenge")
735 }
736 if err := validateEffectivePublicTLSReadiness(si.cfg, effective); err != nil {
737 return err
738 }
739
740 store, err := acmestore.New(filepath.Join(resolveDataDir(si.cfg.Server.DataDir), "acme"))
741 if err != nil {
742 return log.WrapErr(err, "create ACME store")
743 }
744
745 challenges := publictls.NewHTTP01Challenges()
746
747 var zoneResolver *acmelego.CloudflareZoneResolver
748 if effective.Mode == domain.ACMEChallengeCloudflareDNS01 {
749 zoneResolver = acmelego.NewCloudflareZoneResolver(effective.Token)
750 }
751
752 issuer, err := acmelego.NewIssuer(acmelego.Config{
753 Email: si.cfg.TLS.ACME.Email,
754 Challenge: effective.Mode,
755 Token: effective.Token,
756 Store: store,
757 HTTPChallengeSink: challenges,
758 DNSResolvers: publicTLSCfg.DNS.Resolvers,
759 DNSPropagationTimeout: publicTLSCfg.DNS.PropagationTimeout,
760 DNSPollingInterval: publicTLSCfg.DNS.PollingInterval,
761 })
762 if err != nil {
763 return log.WrapErr(err, "create ACME issuer")
764 }
765
766 svc := publictls.NewService(publicTLSCfg, publictls.ServiceDeps{
767 Routes: si.appRoutes(),
768 Issuer: issuer,
769 Store: store,
770 ZoneResolver: zoneResolver,
771 Challenges: challenges,
772 Effective: effective,
773 AdditionalHosts: []string{si.cfg.Server.GordonDomain},
774 })
775
776 if err := svc.Load(ctx); err != nil {
777 log.Warn().Err(err).Msg("failed to load ACME certificates, continuing")
778 }
779
780 log.Info().
781 Str("email", si.cfg.TLS.ACME.Email).
782 Str("challenge", string(effective.Mode)).
783 Msg("public ACME TLS initialized (runtime start deferred)")
784
785 si.svc.publicTLSSvc = svc
786 si.svc.publicTLSRuntime = svc
787 return nil
788}
789
790// publicTLSRuntime is the subset of in.PublicTLSService needed at server runtime
791// after the HTTP listener is bound: reconcile missing certs and start the renewal
792// loop. It is nil when ACME is disabled.
793type publicTLSRuntime interface {
794 Reconcile(context.Context) error
795 StartRenewalLoop(context.Context, time.Duration) <-chan struct{}
796}
797
798func startPublicTLSRuntime(ctx context.Context, svc publicTLSRuntime, log zerowrap.Logger) error {
799 if svc == nil {
800 return nil
801 }
802 reconcileErr := svc.Reconcile(ctx)
803 svc.StartRenewalLoop(ctx, time.Hour)
804 log.Info().Msg("public ACME TLS runtime started")
805 return reconcileErr
806}
807
808// initSecrets validates the secrets backend and creates the app service
809// secret provider.
810func (si *serviceInit) initSecrets() error {
811 backend, err := resolveSecretsBackend(si.cfg.Auth.SecretsBackend)
812 if err != nil {
813 return si.log.WrapErr(err, "failed to resolve secrets backend")
814 }
815 si.svc.serviceSecretProvider = createStandaloneServiceSecretProvider(backend, resolveDataDir(si.cfg.Server.DataDir), si.log)
816 return nil
817}
818
819// initRuntimeAndProxy creates container, backup, registry, image, volume, and proxy services.
820func (si *serviceInit) initRuntimeProxyAndTraffic() error {
821 if err := si.initRuntimeAndProxy(); err != nil {
822 return err
823 }
824 si.svc.trafficManager = trafficadapter.NewManager()
825 return si.initApps()
826}
827
828func containerResourceLimits(cfg Config) (deployment.ResourceLimits, error) {
829 limits := deployment.ResourceLimits{PidsLimit: cfg.Containers.PidsLimit}
830 if cfg.Containers.MemoryLimit != "" {
831 parsed, err := bytesize.Parse(cfg.Containers.MemoryLimit)
832 if err != nil {
833 return limits, fmt.Errorf("invalid containers.memory_limit %q: %w", cfg.Containers.MemoryLimit, err)
834 }
835 limits.MemoryBytes = parsed
836 }
837 if cfg.Containers.CPULimit > 0 {
838 limits.NanoCPUs = int64(cfg.Containers.CPULimit * float64(time.Second))
839 }
840 return limits, nil
841}
842
843// appDeployEngine is the deployment-engine subset the daemon-owned app
844// service executes through. It mirrors the usecase's own port so the
845// composition root stays decoupled from engine internals.
846type appDeployEngine interface {
847 StartDeploy(ctx context.Context, input deployment.DeployInput) (*deployment.StartDeployResult, error)
848 AbandonDeploy(ctx context.Context, claim deployment.DeployClaim) error
849 ExecuteDeploy(ctx context.Context, claim deployment.DeployClaim) (*deployment.DeployResult, error)
850 Stop(ctx context.Context, app, opID string) (*deployment.LifecycleResult, error)
851 Start(ctx context.Context, app, opID string) (*deployment.LifecycleResult, error)
852 Restart(ctx context.Context, app, service, opID string) (*deployment.LifecycleResult, error)
853 Remove(ctx context.Context, app, opID string) (*deployment.LifecycleResult, error)
854}
855
856// newAppDaemonService builds the daemon-owned app administration service.
857// ctx is the daemon/supervisor lifecycle context: background deploy
858// executions derive from it and are cancelled by Shutdown during graceful
859// teardown. It is deliberately never an HTTP request context, so request
860// cancellation cannot abort an in-flight replacement.
861func newAppDaemonService(ctx context.Context, store out.AppState, deploy appDeployEngine, secrets out.SecretWriter, log zerowrap.Logger) *apps.AppServiceImpl {
862 return apps.NewAppServiceImpl(store, deploy, secrets, log).WithDaemonContext(ctx)
863}
864
865// initApps wires the single v3 app engine: bbolt state, image digests,
866// pass-backed secrets, deployment/lifecycle execution, and traffic
867// activation. The daemon is the sole app-state writer; CLI reaches the
868// engine only through the admin /apps surface.
869func (si *serviceInit) initApps() error {
870 dataDir := resolveDataDir(si.cfg.Server.DataDir)
871 store, err := appstate.NewStore(dataDir, si.log)
872 if err != nil {
873 return si.log.WrapErr(err, "failed to open app state")
874 }
875 store.WithRevisionRetention(si.cfg.Apps.RevisionRetention)
876 si.svc.appState = store
877 observeManagedContainers(si.svc.metrics, store, si.log)
878 // One process-wide GC barrier owns the ordering between resource
879 // acquisition/publication and destructive prune.
880 si.svc.gcBarrier = newGCBarrier()
881 if si.svc.backupSvc != nil {
882 si.svc.backupSvc.WithAppState(store)
883 }
884 if si.svc.volumeBackupSvc != nil {
885 si.svc.volumeBackupSvc.WithAppState(store)
886 }
887 if si.svc.imageSvc != nil {
888 si.svc.imageSvc.WithPrunePorts(store, si.svc.runtime, si.svc.gcBarrier)
889 }
890 if si.svc.volumeSvc != nil {
891 si.svc.volumeSvc.WithPrunePorts(store, si.svc.runtime, si.svc.gcBarrier)
892 }
893 registryDomain := si.svc.configSvc.GetRegistryDomain()
894 imagePolicy := domain.ImageSourcePolicy{
895 AllowedRegistries: si.cfg.Images.AllowedRegistries,
896 RequireDigest: si.cfg.Images.RequireDigest,
897 InstallationRegistry: registryDomain,
898 }
899 if err := imagePolicy.Validate(); err != nil {
900 return fmt.Errorf("invalid image registry policy: %w", err)
901 }
902 resolver := imageref.NewResolver(registryDomain, si.svc.manifestStorage, nil).WithPolicy(imagePolicy)
903 // The publisher is the single serialized HTTP/L4 boundary: raw
904 // activations, fail-closed recovery withdrawal, and reload all go
905 // through it. It resolves writers/permission paths late-bound.
906 si.svc.appTrafficPublisher = newAppTrafficPublisher(si.svc, si.cfg)
907 limits, err := containerResourceLimits(si.cfg)
908 if err != nil {
909 return err
910 }
911 si.svc.appDeploySvc = deployment.NewService(deployment.Deps{
912 State: store,
913 Runtime: si.svc.runtime,
914 Images: resolver,
915 Secrets: si.svc.serviceSecretProvider,
916 Registry: deployment.RegistryConfig{
917 Domain: registryDomain,
918 PullAddress: net.JoinHostPort("127.0.0.1", strconv.Itoa(si.svc.configSvc.GetRegistryPort())),
919 Username: si.svc.internalRegUser,
920 Password: si.svc.internalRegPass,
921 },
922 Networks: deployment.NetworkConfig{
923 Prefix: si.cfg.NetworkIsolation.Prefix,
924 Internal: si.cfg.NetworkIsolation.Internal,
925 },
926 Limits: limits,
927 ImagePolicy: imagePolicy,
928 Traffic: si.svc.appTrafficPublisher,
929 }, si.log).WithGCBarrier(si.svc.gcBarrier)
930 si.svc.appActivator = apptraffic.NewActivator(si.log)
931 mountPolicies, err := buildAppMountPolicies(si.cfg)
932 if err != nil {
933 return err
934 }
935 devicePolicies, err := buildAppDevicePolicies(si.cfg)
936 if err != nil {
937 return err
938 }
939 si.svc.appDeploySvc.SetBindPolicies(mountPolicies)
940 si.svc.appDeploySvc.SetDevicePolicies(devicePolicies)
941 appSvcImpl := newAppDaemonService(si.ctx, store, si.svc.appDeploySvc, appsecrets.NewStore(si.log), si.log).
942 WithEntrypoints(appEntrypointListeners(si.cfg)).
943 WithGCBarrier(si.svc.gcBarrier).
944 WithImagePolicy(imagePolicy).
945 WithBindPolicies(mountPolicies).
946 WithDevicePolicies(devicePolicies)
947 si.svc.appSvcImpl = appSvcImpl
948 si.svc.appSvc = appSvcImpl
949 // Health checks resolve from ACTIVE state (loopback backends).
950 si.svc.healthSvc = health.NewService(store, si.svc.runtime, httpprober.New(), si.log)
951 // ACTIVE-derived host index for the proxy: rebuilt after every
952 // activation and at boot. Wired here (store is open); the proxy
953 // itself was built earlier in initRuntimeAndProxy.
954 si.svc.appHostIndex = apptraffic.NewHostIndex()
955 if si.svc.proxySvc != nil {
956 si.svc.proxySvc.WithAppTargets(si.svc.appHostIndex)
957 }
958 return nil
959}
960
961// appRouteSource adapts the ACTIVE-derived host index plus installation
962// external routes to the route-source interfaces (PKI AppRoutes,
963// public-TLS RouteSource). bbolt stays authoritative; this adapter only
964// projects. Fail-closed: unwired index yields zero app hosts. The index
965// is late-bound (built after PKI/public-TLS), so it resolves per call.
966type appRouteSource struct {
967 index func() *apptraffic.HostIndex
968 external func() map[string]string
969}
970
971// AppHosts implements out.AppHostSource.
972func (s *appRouteSource) AppHosts() []out.AppHost {
973 if s == nil || s.index == nil {
974 return nil
975 }
976 index := s.index()
977 if index == nil {
978 return nil
979 }
980 return index.AppHosts()
981}
982
983// GetExternalRoutes implements the installation half of out.AppRoutes
984// and the public-TLS RouteSource.
985func (s *appRouteSource) GetExternalRoutes() map[string]string {
986 if s == nil || s.external == nil {
987 return nil
988 }
989 return s.external()
990}
991
992// GetRoutes implements the public-TLS RouteSource as a consumer-local
993// adapter: host-only entries, Image explicitly unused (no app meaning).
994func (s *appRouteSource) GetRoutes(context.Context) []domain.Route {
995 hosts := s.AppHosts()
996 routes := make([]domain.Route, 0, len(hosts))
997 for _, h := range hosts {
998 routes = append(routes, domain.Route{Domain: h.Host})
999 }
1000 return routes
1001}
1002
1003// appRoutes returns the shared ACTIVE-derived route source, late-bound
1004// to the host index built in initApps.
1005func (si *serviceInit) appRoutes() *appRouteSource {
1006 return &appRouteSource{
1007 index: func() *apptraffic.HostIndex { return si.svc.appHostIndex },
1008 external: func() map[string]string { return si.svc.configSvc.GetExternalRoutes() },
1009 }
1010}
1011
1012// appEntrypointPolicies maps installation entrypoints to the projection
1013// policy (trusted CIDRs, raw-fallback behavior, transport limits).
1014func appEntrypointPolicies(cfg Config) map[string]apptraffic.EntrypointPolicy {
1015 policies := make(map[string]apptraffic.EntrypointPolicy, len(cfg.EntryPoints))
1016 for name, entry := range cfg.EntryPoints {
1017 policies[name] = apptraffic.EntrypointPolicy{
1018 Name: name,
1019 Address: entry.Address,
1020 Protocol: entry.Protocol,
1021 TrustedCIDRs: entry.TrustedCIDRs,
1022 RawFallback: entry.RawFallback,
1023 RawFallbackTrustedCIDRs: entry.RawFallbackTrustedCIDRs,
1024 AllowPublicRawFallback: entry.AllowPublicRawFallback,
1025 }
1026 }
1027 return policies
1028}
1029
1030// appEntrypointListeners maps installation entrypoints to the canonical
1031// listener an app L4 publish declaration must match exactly.
1032func appEntrypointListeners(cfg Config) map[string]domain.EntryPointListener {
1033 listeners := make(map[string]domain.EntryPointListener, len(cfg.EntryPoints))
1034 for name, entry := range cfg.EntryPoints {
1035 listeners[name] = domain.EntryPointListener{Address: entry.Address, Protocol: entry.Protocol}
1036 }
1037 return listeners
1038}
1039
1040// rebuildAppHostIndex re-projects ACTIVE state into the proxy host index
1041// and applies the full HTTP/L4 traffic graph through the serialized
1042// publisher. Call after every activation (deploy/start/restart/stop/
1043// remove) and at boot; callers decide whether a failure is fatal to the
1044// mutation.
1045func rebuildAppHostIndex(ctx context.Context, svc *services) error {
1046 if svc.appTrafficPublisher == nil {
1047 return nil
1048 }
1049 return svc.appTrafficPublisher.RebuildTraffic(ctx)
1050}
1051
1052func (si *serviceInit) initRuntimeAndProxy() error {
1053 var err error
1054
1055 if si.svc.containerSvc, err = createContainerService(si.ctx, si.v, si.cfg, si.svc, si.log); err != nil {
1056 return err
1057 }
1058
1059 if si.svc.backupStorage, si.svc.backupSvc, err = createBackupService(si.cfg, si.svc, si.log); err != nil {
1060 return err
1061 }
1062 if si.svc.volumeBackupStore, si.svc.volumeBackupSvc, si.svc.volumeBackupCfg, err = createVolumeBackupService(si.ctx, si.cfg, si.svc, si.log); err != nil {
1063 return err
1064 }
1065
1066 registryState := registrystate.New()
1067 si.svc.registrySvc = registrySvc.NewService(si.svc.blobStorage, si.svc.manifestStorage, si.svc.eventBus, registryState)
1068 si.svc.imageSvc = images.NewService(si.svc.runtime, si.svc.manifestStorage, si.svc.blobStorage, si.log, registryState)
1069 si.svc.volumeSvc = volumesSvc.NewService(si.svc.runtime)
1070
1071 injectTelemetryMetrics(si.cfg, si.svc, si.log)
1072
1073 proxyCfg, err := buildProxyConfig(si.cfg, si.log)
1074 if err != nil {
1075 return err
1076 }
1077 si.svc.maxBlobChunkSize = proxyCfg.maxBlobChunkSize
1078 si.svc.maxBlobSize = proxyCfg.maxBlobSize
1079 si.svc.proxySvc = proxy.NewService(si.svc.configSvc, proxyCfg.proxyConfig)
1080
1081 // Wire synchronous proxy cache invalidation for zero-downtime deployments.
1082 return nil
1083}
1084
1085// initHandlers creates the auth, health, log, preview, and admin handlers.
1086func (si *serviceInit) registerReloadCoordinatorHooks() {
1087 if si.svc.reloadCoordinator == nil {
1088 return
1089 }
1090 if si.svc.appSvcImpl != nil {
1091 si.svc.reloadCoordinator.SetAppMountPoliciesApplier(func(_ context.Context, reloadCfg Config) error {
1092 policies, err := buildAppMountPolicies(reloadCfg)
1093 if err != nil {
1094 return err
1095 }
1096 devicePolicies, err := buildAppDevicePolicies(reloadCfg)
1097 if err != nil {
1098 return err
1099 }
1100 si.svc.appSvcImpl.SetBindPolicies(policies)
1101 si.svc.appSvcImpl.SetDevicePolicies(devicePolicies)
1102 if si.svc.appDeploySvc != nil {
1103 si.svc.appDeploySvc.SetBindPolicies(policies)
1104 si.svc.appDeploySvc.SetDevicePolicies(devicePolicies)
1105 }
1106 return nil
1107 })
1108 }
1109 if si.svc.containerSvc == nil {
1110 return
1111 }
1112
1113 si.svc.reloadCoordinator.SetContainerConfigApplier(func(reloadCtx context.Context, reloadCfg Config) error {
1114 containerCfg, err := buildContainerServiceConfig(reloadCtx, si.v, reloadCfg, si.svc, si.log)
1115 if err != nil {
1116 return err
1117 }
1118 managementHosts := []string{reloadCfg.Server.GordonDomain}
1119 if si.svc.pkiSvc != nil {
1120 si.svc.pkiSvc.SetAdditionalDomains(managementHosts)
1121 }
1122 if si.svc.publicTLSSvc != nil {
1123 si.svc.publicTLSSvc.SetAdditionalHosts(reloadCtx, managementHosts)
1124 }
1125 var tlsConfig *tls.Config
1126 if hasTLSCapableEntrypoint(reloadCfg) && si.svc.httpsProxyHandler != nil {
1127 var tlsErr error
1128 tlsConfig, tlsErr = proxyTLSConfig(reloadCfg, si.svc.pkiSvc, si.svc.publicTLSSvc, si.log)
1129 if tlsErr != nil {
1130 return tlsErr
1131 }
1132 }
1133 if err := si.svc.appTrafficPublisher.RebuildWithConfig(reloadCtx, reloadCfg); err != nil {
1134 return err
1135 }
1136 si.svc.tlsHTTPEntryPoints = registerTLSMuxHTTPServers(si.svc.trafficManager, reloadCfg, si.svc.httpsProxyHandler, tlsConfig, si.svc.tlsHTTPEntryPoints)
1137 si.svc.smartHTTPEntryPoints = registerSmartTCPHTTPServers(si.svc.trafficManager, reloadCfg, si.svc.httpProxyHandler, si.svc.httpsProxyHandler, tlsConfig, si.svc.smartHTTPEntryPoints)
1138 si.svc.containerSvc.UpdateConfig(containerCfg)
1139 return nil
1140 })
1141}
1142
1143func (si *serviceInit) initHandlers() {
1144 if si.svc.authSvc != nil {
1145 internalAuth := authhandler.InternalAuth{
1146 Username: si.svc.internalRegUser,
1147 Password: si.svc.internalRegPass,
1148 }
1149 si.svc.authHandler = authhandler.NewHandler(si.svc.authSvc, internalAuth, si.log)
1150 }
1151
1152 si.svc.logSvc = logs.NewService(resolveLogFilePath(si.cfg), si.cfg.Logging.File.Enabled, si.svc.runtime, si.log)
1153 // initApps (via initRuntimeProxyAndTraffic) runs before initHandlers,
1154 // so durable ACTIVE state is already open here.
1155 if si.svc.appState != nil {
1156 si.svc.logSvc.WithAppState(si.svc.appState)
1157 }
1158
1159 if si.svc.trafficManager == nil {
1160 si.svc.trafficManager = trafficadapter.NewManager()
1161 }
1162
1163 si.svc.adminHandler = admin.NewHandler(admin.HandlerDeps{
1164 ConfigSvc: si.svc.configSvc,
1165 AuthSvc: si.svc.authSvc,
1166 ContainerSvc: si.svc.containerSvc,
1167 HealthSvc: si.svc.healthSvc,
1168 LogSvc: si.svc.logSvc,
1169 RegistrySvc: si.svc.registrySvc,
1170 ReloadTrigger: si.svc.reloadCoordinator,
1171 Log: si.log,
1172 BackupSvc: si.svc.backupSvc,
1173 VolumeBackupSvc: si.svc.volumeBackupSvc,
1174 ImageSvc: si.svc.imageSvc,
1175 VolumeSvc: si.svc.volumeSvc,
1176 PublicTLSSvc: si.svc.publicTLSSvc,
1177 TrafficSvc: si.svc.trafficManager,
1178 AppSvc: si.svc.appSvc,
1179 })
1180}
1181
1182// injectTelemetryMetrics creates and injects OTel metrics into services when
1183// telemetry is enabled. Skipped otherwise to avoid unnecessary allocations.
1184func injectTelemetryMetrics(cfg Config, svc *services, log zerowrap.Logger) {
1185 if !cfg.Telemetry.Enabled || !cfg.Telemetry.Metrics {
1186 return
1187 }
1188 gordonMetrics, err := telemetry.NewMetrics()
1189 if err != nil {
1190 log.Warn().Err(err).Msg("failed to create telemetry metrics, continuing without metrics")
1191 return
1192 }
1193 svc.registrySvc.SetMetrics(gordonMetrics)
1194 svc.eventBus.SetMetrics(gordonMetrics)
1195 svc.metrics = gordonMetrics
1196}
1197
1198// observeManagedContainers exports the managed-container gauge from app
1199// state. No-op when metrics are disabled.
1200func observeManagedContainers(metrics *telemetry.Metrics, state out.AppStateReader, log zerowrap.Logger) {
1201 if metrics == nil {
1202 return
1203 }
1204 if err := metrics.ObserveManagedContainers(func(ctx context.Context) (int64, error) {
1205 return apps.CountManagedContainers(ctx, state)
1206 }); err != nil {
1207 log.Warn().Err(err).Msg("failed to register managed container gauge")
1208 }
1209}
1210
1211func setupInternalRegistryAuth(svc *services, log zerowrap.Logger) error {
1212 var err error
1213 svc.internalRegUser, svc.internalRegPass, err = generateInternalRegistryAuth()
1214 if err != nil {
1215 return log.WrapErr(err, "failed to generate internal registry credentials")
1216 }
1217
1218 // Persist credentials to file for CLI access (gordon auth internal)
1219 if err := persistInternalCredentials(svc.internalRegUser, svc.internalRegPass); err != nil {
1220 log.Warn().Err(err).Msg("failed to persist internal credentials for CLI access")
1221 }
1222
1223 log.Debug().Msg("internal registry auth generated for loopback pulls")
1224 return nil
1225}
1226
1227// resolveLogFilePath returns the configured log file path or a default.
1228func resolveLogFilePath(cfg Config) string {
1229 if cfg.Logging.File.Path != "" {
1230 return cfg.Logging.File.Path
1231 }
1232 dataDir := cfg.Server.DataDir
1233 if dataDir == "" {
1234 dataDir = DefaultDataDir()
1235 }
1236 return filepath.Join(dataDir, "logs", "gordon.log")
1237}
1238
1239// resolveRuntimeConfig converts a server.runtime config value to a socket path.
1240// "auto" or "" means auto-detect.
1241// Named runtimes ("podman", "docker") are resolved to well-known socket paths.
1242// URI schemes (unix://) are stripped so callers receive a bare path.
1243func resolveRuntimeConfig(value string) string {
1244 if value == "" || value == "auto" {
1245 return ""
1246 }
1247 // Named runtimes: resolve to well-known socket paths.
1248 switch value {
1249 case "podman":
1250 // Check XDG_RUNTIME_DIR first (rootless Podman).
1251 if xdg := os.Getenv("XDG_RUNTIME_DIR"); xdg != "" {
1252 candidate := filepath.Join(xdg, "podman", "podman.sock")
1253 if _, err := os.Stat(candidate); err == nil {
1254 return candidate
1255 }
1256 }
1257 // Fallback to system-wide Podman socket.
1258 return "/run/podman/podman.sock"
1259 case "docker":
1260 return "/var/run/docker.sock"
1261 }
1262 // Explicit socket path — strip URI scheme if present.
1263 if socketPath, ok := strings.CutPrefix(value, "unix://"); ok {
1264 return socketPath
1265 }
1266 return value
1267}
1268
1269// createOutputAdapters creates the container runtime and event bus.
1270func createOutputAdapters(ctx context.Context, log zerowrap.Logger, runtimeSocket string) (*docker.Runtime, *eventbus.InMemory, error) {
1271 detection := docker.DetectRuntimeSocket(runtimeSocket)
1272
1273 var runtime *docker.Runtime
1274 var err error
1275
1276 switch detection.Source {
1277 case "none":
1278 return nil, nil, fmt.Errorf("no container runtime found: checked Docker socket, Podman socket, DOCKER_HOST env var. Install Docker or Podman, or set server.runtime in config")
1279 case "DOCKER_HOST_passthrough":
1280 detection.RuntimeName = "docker"
1281 runtime, err = docker.NewRuntime()
1282 default:
1283 if detection.SocketPath != "" {
1284 runtime, err = docker.NewRuntimeWithSocket(detection.SocketPath)
1285 } else {
1286 runtime, err = docker.NewRuntime()
1287 }
1288 }
1289 if err != nil {
1290 return nil, nil, log.WrapErr(err, "failed to create container runtime")
1291 }
1292
1293 if err := runtime.Ping(ctx); err != nil {
1294 return nil, nil, log.WrapErr(err, fmt.Sprintf("container runtime not available (detected: %s via %s)", detection.RuntimeName, detection.Source))
1295 }
1296
1297 runtimeVersion, _ := runtime.Version(ctx)
1298 log.Info().
1299 Str("runtime", detection.RuntimeName).
1300 Str("version", runtimeVersion).
1301 Str("source", detection.Source).
1302 Msg("container runtime initialized")
1303
1304 eventBus := eventbus.NewInMemory(100, log)
1305
1306 return runtime, eventBus, nil
1307}
1308
1309// createStorage creates blob and manifest storage.
1310func createStorage(cfg Config, log zerowrap.Logger) (*filesystem.BlobStorage, *filesystem.ManifestStorage, error) {
1311 dataDir := cfg.Server.DataDir
1312 if dataDir == "" {
1313 dataDir = DefaultDataDir()
1314 }
1315
1316 registryDir := filepath.Join(dataDir, "registry")
1317
1318 blobStorage, err := filesystem.NewBlobStorage(registryDir, log)
1319 if err != nil {
1320 return nil, nil, log.WrapErr(err, "failed to create blob storage")
1321 }
1322
1323 manifestStorage, err := filesystem.NewManifestStorage(registryDir, log)
1324 if err != nil {
1325 return nil, nil, log.WrapErr(err, "failed to create manifest storage")
1326 }
1327
1328 return blobStorage, manifestStorage, nil
1329}
1330
1331// createLogWriter creates the container log writer.
1332func createLogWriter(cfg Config, log zerowrap.Logger) (*logwriter.LogWriter, error) {
1333 if !cfg.Logging.ContainerLogs.Enabled {
1334 log.Debug().Msg("container log collection disabled")
1335 return nil, nil
1336 }
1337
1338 // Determine log directory
1339 logDir := cfg.Logging.ContainerLogs.Dir
1340 if logDir == "" {
1341 dataDir := cfg.Server.DataDir
1342 if dataDir == "" {
1343 dataDir = DefaultDataDir()
1344 }
1345 logDir = filepath.Join(dataDir, "logs", "containers")
1346 }
1347
1348 writer, err := logwriter.New(logwriter.Config{
1349 Dir: logDir,
1350 MaxSize: cfg.Logging.ContainerLogs.MaxSize,
1351 MaxBackups: cfg.Logging.ContainerLogs.MaxBackups,
1352 MaxAge: cfg.Logging.ContainerLogs.MaxAge,
1353 })
1354 if err != nil {
1355 return nil, log.WrapErr(err, "failed to create container log writer")
1356 }
1357
1358 log.Info().Str("dir", logDir).Msg("container log collection enabled")
1359 return writer, nil
1360}
1361
1362const (
1363 internalRegistryUsername = "gordon-internal"
1364 serviceTokenSubject = "gordon-service"
1365 serviceTokenDefaultTTL = 30 * 24 * time.Hour
1366)
1367
1368func generateInternalRegistryAuth() (string, string, error) {
1369 password, err := randomTokenHex(32)
1370 if err != nil {
1371 return "", "", err
1372 }
1373 return internalRegistryUsername, password, nil
1374}
1375
1376func randomTokenHex(size int) (string, error) {
1377 buf := make([]byte, size)
1378 if _, err := rand.Read(buf); err != nil {
1379 return "", err
1380 }
1381 return hex.EncodeToString(buf), nil
1382}
1383
1384// InternalCredentials holds the internal registry credentials for CLI access.
1385type InternalCredentials struct {
1386 Username string `json:"username"`
1387 Password string `json:"password"`
1388}
1389
1390// getSecureRuntimeDir returns a secure directory for runtime files.
1391// Priority: XDG_RUNTIME_DIR > ~/.gordon/run
1392func getSecureRuntimeDir() (string, error) {
1393 // Try XDG_RUNTIME_DIR first (typically /run/user/<uid> on Linux)
1394 if runtimeDir := os.Getenv("XDG_RUNTIME_DIR"); runtimeDir != "" {
1395 gordonDir := filepath.Join(runtimeDir, "gordon")
1396 if err := os.MkdirAll(gordonDir, 0700); err == nil {
1397 return gordonDir, nil
1398 }
1399 }
1400
1401 // Fall back to ~/.gordon/run
1402 homeDir, err := os.UserHomeDir()
1403 if err != nil {
1404 return "", fmt.Errorf("failed to get home directory: %w", err)
1405 }
1406
1407 gordonDir := filepath.Join(homeDir, ".gordon", "run")
1408 if err := os.MkdirAll(gordonDir, 0700); err != nil {
1409 return "", fmt.Errorf("failed to create runtime directory: %w", err)
1410 }
1411
1412 return gordonDir, nil
1413}
1414
1415// getInternalCredentialsFile returns the path to the internal credentials file.
1416// SECURITY: Credentials are stored in a secure location with restricted permissions.
1417func getInternalCredentialsFile() string {
1418 runtimeDir, err := getSecureRuntimeDir()
1419 if err != nil {
1420 // Fall back to temp dir if we can't get secure dir (shouldn't happen)
1421 return filepath.Join(os.TempDir(), "gordon-internal-creds.json")
1422 }
1423 return filepath.Join(runtimeDir, "internal-creds.json")
1424}
1425
1426// persistInternalCredentials saves the internal registry credentials to a secure file.
1427// SECURITY: Credentials are stored in XDG_RUNTIME_DIR or ~/.gordon/run with 0600 permissions.
1428// The file is cleaned up on graceful shutdown but may persist if Gordon crashes.
1429// These credentials are for internal loopback communication only and are regenerated on each start.
1430func persistInternalCredentials(username, password string) error {
1431 creds := InternalCredentials{
1432 Username: username,
1433 Password: password,
1434 }
1435 data, err := json.Marshal(creds)
1436 if err != nil {
1437 return fmt.Errorf("failed to marshal credentials: %w", err)
1438 }
1439
1440 credFile := getInternalCredentialsFile()
1441
1442 // Ensure parent directory exists with secure permissions
1443 if err := os.MkdirAll(filepath.Dir(credFile), 0700); err != nil {
1444 return fmt.Errorf("failed to create credentials directory: %w", err)
1445 }
1446
1447 // Write file with restrictive permissions (owner read/write only)
1448 if err := os.WriteFile(credFile, data, 0600); err != nil {
1449 return fmt.Errorf("failed to write credentials file: %w", err)
1450 }
1451 return nil
1452}
1453
1454// cleanupInternalCredentials removes the internal credentials file.
1455func cleanupInternalCredentials() {
1456 _ = os.Remove(getInternalCredentialsFile())
1457}
1458
1459// getInternalCredentialsCandidates returns candidate file paths in priority order:
1460// 1. XDG_RUNTIME_DIR/gordon/ (set by systemd for the daemon)
1461// 2. /run/user/<uid>/gordon/ (well-known systemd default, for CLI in shells without XDG_RUNTIME_DIR)
1462// 3. ~/.gordon/run/ (fallback for non-systemd environments)
1463// 4. os.TempDir() (last resort, matches getInternalCredentialsFile fallback path)
1464func getInternalCredentialsCandidates() []string {
1465 var candidates []string
1466
1467 // 1. XDG_RUNTIME_DIR (set in daemon's environment)
1468 if runtimeDir := os.Getenv("XDG_RUNTIME_DIR"); runtimeDir != "" {
1469 candidates = append(candidates, filepath.Join(runtimeDir, "gordon", "internal-creds.json"))
1470 }
1471
1472 // 2. /run/user/<uid>/gordon/ (systemd default, may not be in CLI's env)
1473 uid := os.Getuid()
1474 sysRuntime := filepath.Join("/run/user", fmt.Sprintf("%d", uid), "gordon", "internal-creds.json")
1475 // Avoid duplicate if XDG_RUNTIME_DIR already points here
1476 if len(candidates) == 0 || candidates[0] != sysRuntime {
1477 candidates = append(candidates, sysRuntime)
1478 }
1479
1480 // 3. ~/.gordon/run/ fallback
1481 if homeDir, err := os.UserHomeDir(); err == nil {
1482 candidates = append(candidates, filepath.Join(homeDir, ".gordon", "run", "internal-creds.json"))
1483 }
1484
1485 // 4. os.TempDir() last resort — matches the fallback path in getInternalCredentialsFile,
1486 // ensuring GetInternalCredentials can find credentials even when getSecureRuntimeDir fails.
1487 candidates = append(candidates, filepath.Join(os.TempDir(), "gordon-internal-creds.json"))
1488
1489 return candidates
1490}
1491
1492// GetInternalCredentialsFromCandidates reads credentials from the first candidate file that exists.
1493// Exported for testing.
1494func GetInternalCredentialsFromCandidates(candidates []string) (*InternalCredentials, error) {
1495 var lastErr error
1496 for _, path := range candidates {
1497 data, err := os.ReadFile(path)
1498 if os.IsNotExist(err) {
1499 continue
1500 }
1501 if err != nil {
1502 // Non-permission errors (e.g. EACCES) may be transient or path-specific;
1503 // record and try the next candidate rather than failing immediately.
1504 lastErr = fmt.Errorf("failed to read credentials file %s: %w", path, err)
1505 continue
1506 }
1507 var creds InternalCredentials
1508 if err := json.Unmarshal(data, &creds); err != nil {
1509 // Corrupt file — record and fall through to lower-priority candidates.
1510 lastErr = fmt.Errorf("failed to parse credentials at %s: %w", path, err)
1511 continue
1512 }
1513 return &creds, nil
1514 }
1515 if lastErr != nil {
1516 return nil, lastErr
1517 }
1518 return nil, fmt.Errorf("no credentials file found (is Gordon running?): checked %v", candidates)
1519}
1520
1521// GetInternalCredentials reads the internal registry credentials from file.
1522// Probes all candidate runtime directories so CLI works regardless of whether
1523// XDG_RUNTIME_DIR is set in the current shell environment.
1524func GetInternalCredentials() (*InternalCredentials, error) {
1525 return GetInternalCredentialsFromCandidates(getInternalCredentialsCandidates())
1526}
1527
1528// createAuthService creates the authentication service and token store.
1529func createAuthService(ctx context.Context, cfg Config, log zerowrap.Logger) (out.TokenStore, *auth.Service, error) {
1530 if !cfg.Auth.Enabled {
1531 log.Warn().Msg("auth.enabled=false detected: running in local-only mode (registry loopback-only, admin API disabled)")
1532 return nil, nil, nil
1533 }
1534
1535 authType, err := resolveAuthType(cfg)
1536 if err != nil {
1537 return nil, nil, fmt.Errorf("resolve auth type: %w", err)
1538 }
1539 backend, err := resolveSecretsBackend(cfg.Auth.SecretsBackend)
1540 if err != nil {
1541 return nil, nil, log.WrapErr(err, "failed to resolve secrets backend")
1542 }
1543 dataDir := resolveDataDir(cfg.Server.DataDir)
1544
1545 store, err := createTokenStore(backend, dataDir, log)
1546 if err != nil {
1547 return nil, nil, err
1548 }
1549
1550 authConfig, err := buildAuthConfig(ctx, cfg, authType, backend, dataDir, log)
1551 if err != nil {
1552 return nil, nil, err
1553 }
1554
1555 authSvc := auth.NewService(authConfig, store, log)
1556
1557 log.Info().
1558 Str("type", string(authType)).
1559 Str("backend", string(backend)).
1560 Msg("registry authentication enabled")
1561
1562 return store, authSvc, nil
1563}
1564
1565// resolveAuthType determines the auth type from config.
1566// Token-only authentication is the only supported mode.
1567func resolveAuthType(cfg Config) (domain.AuthType, error) {
1568 if cfg.Auth.Type != "" && cfg.Auth.Type != "token" {
1569 return "", fmt.Errorf("unsupported auth.type %q; only \"token\" is supported", cfg.Auth.Type)
1570 }
1571 return domain.AuthTypeToken, nil
1572}
1573
1574func resolveSecretsBackend(backend string) (domain.SecretsBackend, error) {
1575 switch backend {
1576 case "pass":
1577 return domain.SecretsBackendPass, nil
1578 case "sops":
1579 return domain.SecretsBackendSops, nil
1580 case "unsafe":
1581 return domain.SecretsBackendUnsafe, nil
1582 case "":
1583 return "", fmt.Errorf("auth.secrets_backend is required")
1584 default:
1585 return "", fmt.Errorf("unsupported auth.secrets_backend %q", backend)
1586 }
1587}
1588
1589func resolveDataDir(dataDir string) string {
1590 if dataDir == "" {
1591 return DefaultDataDir()
1592 }
1593 return dataDir
1594}
1595
1596func resolveRegistryDomains(cfg Config) (string, []string) {
1597 registryDomain := cfg.Server.RegistryDomain
1598 if registryDomain == "" {
1599 registryDomain = cfg.Server.GordonDomain
1600 }
1601 return registryDomain, append([]string{}, cfg.Server.LegacyRegistryDomains...)
1602}
1603
1604func createTokenStore(backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) (out.TokenStore, error) {
1605 // Token store is always created since tokens work in both auth modes
1606 store, err := tokenstore.NewStore(backend, dataDir, log)
1607 if err != nil {
1608 return nil, log.WrapErr(err, "failed to create token store")
1609 }
1610 return store, nil
1611}
1612
1613func buildAuthConfig(ctx context.Context, cfg Config, authType domain.AuthType, backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) (auth.Config, error) {
1614 authConfig := auth.Config{
1615 Enabled: cfg.Auth.Enabled,
1616 AuthType: authType,
1617 Username: cfg.Auth.Username,
1618 }
1619
1620 // Token config is always required (tokens work in all auth modes)
1621 secret, expiry, err := loadTokenConfig(ctx, cfg, backend, dataDir, log)
1622 if err != nil {
1623 return auth.Config{}, err
1624 }
1625 authConfig.TokenSecret = secret
1626 authConfig.TokenExpiry = expiry
1627
1628 accessTokenTTL := 15 * time.Minute // default
1629 if cfg.Auth.AccessTokenTTL != "" {
1630 parsed, err := time.ParseDuration(cfg.Auth.AccessTokenTTL)
1631 if err != nil {
1632 return auth.Config{}, fmt.Errorf("invalid auth.access_token_ttl %q: %w", cfg.Auth.AccessTokenTTL, err)
1633 }
1634 if parsed <= 0 {
1635 return auth.Config{}, fmt.Errorf("auth.access_token_ttl must be positive")
1636 }
1637 if parsed > auth.MaxAccessTokenLifetime {
1638 return auth.Config{}, fmt.Errorf("auth.access_token_ttl must not exceed %v", auth.MaxAccessTokenLifetime)
1639 }
1640 accessTokenTTL = parsed
1641 }
1642 authConfig.AccessTokenTTL = accessTokenTTL
1643
1644 return authConfig, nil
1645}
1646
1647func loadTokenConfig(ctx context.Context, cfg Config, backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) ([]byte, time.Duration, error) {
1648 secret, err := loadTokenSecret(ctx, cfg, backend, dataDir, log)
1649 if err != nil {
1650 return nil, 0, err
1651 }
1652
1653 expiry, err := parseTokenExpiry(cfg.Auth.TokenExpiry)
1654 if err != nil {
1655 return nil, 0, err
1656 }
1657
1658 return secret, expiry, nil
1659}
1660
1661// TokenSecretEnvVar is the environment variable for the JWT signing secret.
1662// SECURITY: This takes priority over config file to allow secure secret injection.
1663const TokenSecretEnvVar = "GORDON_AUTH_TOKEN_SECRET" //nolint:gosec // This is an env var name, not a credential
1664
1665func loadTokenSecret(ctx context.Context, cfg Config, backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) ([]byte, error) {
1666 // SECURITY: Priority order for token secret:
1667 // 1. Environment variable (most secure - no disk exposure)
1668 // 2. Secrets backend (pass/sops - encrypted)
1669 // 3. Config file path (least preferred)
1670
1671 const minTokenSecretLength = 32
1672
1673 // Check environment variable first
1674 if envSecret := os.Getenv(TokenSecretEnvVar); envSecret != "" {
1675 if len(envSecret) < minTokenSecretLength {
1676 return nil, fmt.Errorf("token secret from %s must be at least %d bytes (got %d)", TokenSecretEnvVar, minTokenSecretLength, len(envSecret))
1677 }
1678 log.Debug().Msg("using token secret from environment variable")
1679 return []byte(envSecret), nil
1680 }
1681
1682 // Fall back to config-specified path via secrets backend
1683 if cfg.Auth.TokenSecret == "" {
1684 return nil, fmt.Errorf("token_secret is required for JWT token generation; set %s environment variable or configure auth.token_secret", TokenSecretEnvVar)
1685 }
1686
1687 secret, err := loadSecret(ctx, backend, cfg.Auth.TokenSecret, dataDir, log)
1688 if err != nil {
1689 return nil, log.WrapErr(err, "failed to load token secret")
1690 }
1691
1692 if len(secret) < minTokenSecretLength {
1693 return nil, fmt.Errorf("token_secret must be at least %d bytes (got %d); use a strong random secret", minTokenSecretLength, len(secret))
1694 }
1695
1696 return []byte(secret), nil
1697}
1698
1699func parseTokenExpiry(expiry string) (time.Duration, error) {
1700 if expiry == "" {
1701 return 0, nil
1702 }
1703
1704 parsed, err := duration.Parse(expiry)
1705 if err != nil {
1706 return 0, fmt.Errorf("invalid token_expiry: %w", err)
1707 }
1708
1709 return parsed, nil
1710}
1711
1712// loadSecret loads a secret from the configured backend.
1713func loadSecret(ctx context.Context, backend domain.SecretsBackend, path, dataDir string, log zerowrap.Logger) (string, error) {
1714 switch backend {
1715 case domain.SecretsBackendPass:
1716 provider := secrets.NewPassProvider(log)
1717 return provider.GetSecret(ctx, path)
1718 case domain.SecretsBackendSops:
1719 provider := secrets.NewSopsProvider(log)
1720 return provider.GetSecret(ctx, path)
1721 case domain.SecretsBackendUnsafe:
1722 // For unsafe backend, path is relative to dataDir/secrets/.
1723 return readUnsafeSecret(dataDir, path)
1724 default:
1725 return "", fmt.Errorf("unknown secrets backend: %s", backend)
1726 }
1727}
1728
1729func readFileBeneath(root, cleanedRelPath string) ([]byte, error) {
1730 rootFD, err := unix.Open(root, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
1731 if err != nil {
1732 return nil, fmt.Errorf("failed to open secrets root: %w", err)
1733 }
1734 defer unix.Close(rootFD)
1735
1736 parts := strings.Split(filepath.ToSlash(cleanedRelPath), "/")
1737 dirFD := rootFD
1738 var closeDirFDs []int
1739 defer func() {
1740 for i := len(closeDirFDs) - 1; i >= 0; i-- {
1741 _ = unix.Close(closeDirFDs[i])
1742 }
1743 }()
1744
1745 for i, part := range parts {
1746 if part == "" || part == "." || part == ".." {
1747 return nil, fmt.Errorf("invalid secret path: path must stay under dataDir/secrets")
1748 }
1749 last := i == len(parts)-1
1750 if last {
1751 fd, err := unix.Openat(dirFD, part, unix.O_RDONLY|unix.O_NONBLOCK|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
1752 if err != nil {
1753 return nil, fmt.Errorf("failed to read secret file: %w", err)
1754 }
1755 defer unix.Close(fd)
1756 var st unix.Stat_t
1757 if err := unix.Fstat(fd, &st); err != nil {
1758 return nil, fmt.Errorf("failed to stat secret file: %w", err)
1759 }
1760 if st.Mode&unix.S_IFMT != unix.S_IFREG {
1761 return nil, fmt.Errorf("invalid secret path: secret must be a regular file")
1762 }
1763 data, err := os.ReadFile(fmt.Sprintf("/proc/self/fd/%d", fd))
1764 if err != nil {
1765 return nil, fmt.Errorf("failed to read secret file: %w", err)
1766 }
1767 return data, nil
1768 }
1769
1770 nextFD, err := unix.Openat(dirFD, part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
1771 if err != nil {
1772 return nil, fmt.Errorf("failed to open secret path component: %w", err)
1773 }
1774 closeDirFDs = append(closeDirFDs, nextFD)
1775 dirFD = nextFD
1776 }
1777
1778 return nil, fmt.Errorf("invalid secret path: empty path")
1779}
1780
1781type unsafeSecretProvider struct {
1782 dataDir string
1783}
1784
1785func (p unsafeSecretProvider) Name() string { return string(domain.SecretsBackendUnsafe) }
1786
1787func (p unsafeSecretProvider) IsAvailable() bool { return true }
1788
1789func (p unsafeSecretProvider) GetSecret(_ context.Context, path string) (string, error) {
1790 return readUnsafeSecret(p.dataDir, path)
1791}
1792
1793func createStandaloneServiceSecretProvider(backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) out.SecretProvider {
1794 switch backend {
1795 case domain.SecretsBackendPass:
1796 return secrets.NewPassProvider(log)
1797 case domain.SecretsBackendSops:
1798 return secrets.NewSopsProvider(log)
1799 case domain.SecretsBackendUnsafe:
1800 return unsafeSecretProvider{dataDir: dataDir}
1801 default:
1802 return nil
1803 }
1804}
1805
1806func readUnsafeSecret(dataDir, secretPath string) (string, error) {
1807 if filepath.IsAbs(secretPath) {
1808 return "", fmt.Errorf("invalid secret path: absolute paths are not allowed")
1809 }
1810 cleaned := filepath.Clean(secretPath)
1811 if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) {
1812 return "", fmt.Errorf("invalid secret path: path must stay under dataDir/secrets")
1813 }
1814
1815 root := filepath.Clean(filepath.Join(dataDir, "secrets"))
1816 data, err := readFileBeneath(root, cleaned)
1817 if err != nil {
1818 return "", err
1819 }
1820 return string(data), nil
1821}
1822
1823// proxyConfigResult holds parsed proxy and blob chunk size config.
1824type proxyConfigResult struct {
1825 proxyConfig proxy.Config
1826 maxBlobChunkSize int64
1827 maxBlobSize int64
1828}
1829
1830type configWatcher interface {
1831 Watch(ctx context.Context, onChange func()) error
1832}
1833
1834// publicTLSReconciler is the interface for reconciling public TLS certificates.
1835type publicTLSReconciler interface {
1836 Reconcile(context.Context) error
1837}
1838
1839type configReloader interface {
1840 Reload(ctx context.Context) error
1841}
1842
1843type proxyConfigUpdater interface {
1844 UpdateConfig(config proxy.Config)
1845}
1846
1847type reloadTrigger interface {
1848 Trigger(ctx context.Context) error
1849}
1850
1851type loadedConfigApplier interface {
1852 ApplyLoadedConfig(ctx context.Context) error
1853}
1854
1855type reloadCoordinator struct {
1856 mu sync.Mutex
1857 lastRun time.Time
1858 debounce time.Duration
1859 trailingTimer *time.Timer
1860 trailingGeneration uint64
1861 stopped bool
1862
1863 // lifecycleCtx owns the context used by debounced trailing reloads. Its
1864 // base is detached from any caller's cancellation so a short-lived request
1865 // context cannot abort a coalesced apply, while lifecycleCancel lets Stop
1866 // tear the coordinator's own lifecycle down explicitly.
1867 lifecycleCtx context.Context
1868 lifecycleCancel context.CancelFunc
1869
1870 configSvc configReloader
1871 v *viper.Viper
1872 proxySvc proxyConfigUpdater
1873 applyContainerConfig func(context.Context, Config) error
1874 applyAppMountPolicy func(context.Context, Config) error
1875 registryLimits interface {
1876 UpdateBlobLimits(maxBlobChunkSize, maxBlobSize int64)
1877 }
1878 eventBus out.EventPublisher
1879 publicTLS publicTLSReconciler
1880 log zerowrap.Logger
1881}
1882
1883func newReloadCoordinator(v *viper.Viper, configSvc configReloader, proxySvc proxyConfigUpdater, registryLimits interface {
1884 UpdateBlobLimits(maxBlobChunkSize, maxBlobSize int64)
1885}, eventBus out.EventPublisher, publicTLS publicTLSReconciler, log zerowrap.Logger) *reloadCoordinator {
1886 return &reloadCoordinator{
1887 debounce: 500 * time.Millisecond,
1888 configSvc: configSvc,
1889 v: v,
1890 proxySvc: proxySvc,
1891 registryLimits: registryLimits,
1892 eventBus: eventBus,
1893 publicTLS: publicTLS,
1894 log: log,
1895 }
1896}
1897
1898func (c *reloadCoordinator) SetRegistryLimits(limits interface {
1899 UpdateBlobLimits(maxBlobChunkSize, maxBlobSize int64)
1900}) {
1901 c.mu.Lock()
1902 defer c.mu.Unlock()
1903 c.registryLimits = limits
1904}
1905
1906func (c *reloadCoordinator) SetContainerConfigApplier(apply func(context.Context, Config) error) {
1907 c.mu.Lock()
1908 defer c.mu.Unlock()
1909 c.applyContainerConfig = apply
1910}
1911
1912// SetAppMountPoliciesApplier wires the callback that atomically republishes
1913// validated administrative bind policies after a successful reload.
1914func (c *reloadCoordinator) SetAppMountPoliciesApplier(apply func(context.Context, Config) error) {
1915 c.mu.Lock()
1916 defer c.mu.Unlock()
1917 c.applyAppMountPolicy = apply
1918}
1919
1920// Trigger requests a config reload that first re-reads config from disk.
1921func (c *reloadCoordinator) Trigger(ctx context.Context) error {
1922 c.mu.Lock()
1923 defer c.mu.Unlock()
1924
1925 return c.reloadDebouncedLocked(ctx, true)
1926}
1927
1928// ApplyLoadedConfig requests a reload of the config the watcher already
1929// loaded from disk. It shares the debounce/coalescing policy with Trigger so
1930// a burst of fsnotify callbacks applies the final state exactly once.
1931func (c *reloadCoordinator) ApplyLoadedConfig(ctx context.Context) error {
1932 c.mu.Lock()
1933 defer c.mu.Unlock()
1934
1935 return c.reloadDebouncedLocked(ctx, false)
1936}
1937
1938// reloadDebouncedLocked is the single owner of the debounce/coalescing policy
1939// shared by every reload entrypoint. Requests inside the debounce window are
1940// merged into one trailing reload carrying the latest loadConfig intent.
1941func (c *reloadCoordinator) reloadDebouncedLocked(ctx context.Context, loadConfig bool) error {
1942 now := time.Now()
1943 if !c.lastRun.IsZero() && now.Sub(c.lastRun) < c.debounce {
1944 c.scheduleTrailingReloadLocked(ctx, loadConfig)
1945 return nil
1946 }
1947
1948 return c.reloadLocked(ctx, loadConfig)
1949}
1950
1951func (c *reloadCoordinator) scheduleTrailingReloadLocked(ctx context.Context, loadConfig bool) {
1952 if c.stopped {
1953 return
1954 }
1955 if c.trailingTimer != nil {
1956 c.trailingTimer.Stop()
1957 }
1958 if c.lifecycleCancel == nil {
1959 c.lifecycleCtx, c.lifecycleCancel = context.WithCancel(context.WithoutCancel(ctx))
1960 }
1961 c.trailingGeneration++
1962 generation := c.trailingGeneration
1963 trailingCtx := c.lifecycleCtx
1964 c.trailingTimer = time.AfterFunc(c.debounce, func() {
1965 c.runTrailingReload(trailingCtx, generation, loadConfig)
1966 })
1967 c.log.Debug().Msg("coalescing config reload trigger")
1968}
1969
1970func (c *reloadCoordinator) runTrailingReload(ctx context.Context, generation uint64, loadConfig bool) {
1971 c.mu.Lock()
1972 defer c.mu.Unlock()
1973
1974 if c.stopped || generation != c.trailingGeneration {
1975 return
1976 }
1977 c.trailingTimer = nil
1978 if err := c.reloadLocked(ctx, loadConfig); err != nil {
1979 c.log.Error().Err(err).Msg("failed trailing config reload")
1980 }
1981}
1982
1983func (c *reloadCoordinator) Stop() {
1984 c.mu.Lock()
1985 defer c.mu.Unlock()
1986 c.stopped = true
1987 c.trailingGeneration++
1988 if c.trailingTimer != nil {
1989 c.trailingTimer.Stop()
1990 c.trailingTimer = nil
1991 }
1992 if c.lifecycleCancel != nil {
1993 c.lifecycleCancel()
1994 }
1995}
1996
1997func (c *reloadCoordinator) reloadLocked(ctx context.Context, loadConfig bool) error {
1998 now := time.Now()
1999 if loadConfig {
2000 if err := c.configSvc.Reload(ctx); err != nil {
2001 c.log.Error().Err(err).Msg("failed to reload config")
2002 return fmt.Errorf("failed to reload config: %w", err)
2003 }
2004 }
2005
2006 if err := c.applyLoadedConfig(ctx, now); err != nil {
2007 return err
2008 }
2009
2010 return nil
2011}
2012
2013func (c *reloadCoordinator) applyLoadedConfig(ctx context.Context, now time.Time) error {
2014 var reloadCfg Config
2015 if err := c.v.Unmarshal(&reloadCfg); err != nil {
2016 c.log.Error().Err(err).Msg("failed to unmarshal config on reload")
2017 return fmt.Errorf("failed to unmarshal config on reload: %w", err)
2018 }
2019 if err := validateEntrypointMigration(c.v, reloadCfg); err != nil {
2020 return err
2021 }
2022 if err := validateRetiredAppConfig(c.v); err != nil {
2023 return err
2024 }
2025 if err := validateAppPolicies(c.v, reloadCfg); err != nil {
2026 return err
2027 }
2028
2029 reloadedProxy, err := buildProxyConfig(reloadCfg, c.log)
2030 if err != nil {
2031 c.log.Error().Err(err).Msg("failed to parse proxy config on reload")
2032 return fmt.Errorf("failed to parse proxy config on reload: %w", err)
2033 }
2034
2035 if c.applyContainerConfig != nil {
2036 if err := c.applyContainerConfig(ctx, reloadCfg); err != nil {
2037 c.log.Error().Err(err).Msg("failed to apply container config on reload")
2038 return fmt.Errorf("failed to apply container config on reload: %w", err)
2039 }
2040 }
2041 // Publish the reloaded bind policies only after conversion and shape
2042 // validation succeed, so a bad edit keeps the previous policies live.
2043 if c.applyAppMountPolicy != nil {
2044 if err := c.applyAppMountPolicy(ctx, reloadCfg); err != nil {
2045 c.log.Error().Err(err).Msg("failed to apply app mount policies on reload")
2046 return fmt.Errorf("failed to apply app mount policies on reload: %w", err)
2047 }
2048 }
2049 c.proxySvc.UpdateConfig(reloadedProxy.proxyConfig)
2050 if c.registryLimits != nil {
2051 c.registryLimits.UpdateBlobLimits(reloadedProxy.maxBlobChunkSize, reloadedProxy.maxBlobSize)
2052 }
2053
2054 // Reconcile public TLS before publishing reload events so certificate
2055 // authorization reflects the loaded config even if event delivery fails.
2056 // A transient ACME issue must not abort the rest of the reload.
2057 if c.publicTLS != nil {
2058 if err := c.publicTLS.Reconcile(ctx); err != nil {
2059 c.log.Warn().Err(err).Msg("failed to reconcile public TLS certificates after reload, continuing")
2060 }
2061 }
2062
2063 if c.eventBus != nil {
2064 if err := c.eventBus.Publish(domain.EventConfigReload, nil); err != nil {
2065 c.log.Error().Err(err).Msg("failed to publish config reload event")
2066 return fmt.Errorf("failed to publish config reload event: %w", err)
2067 }
2068 }
2069
2070 c.lastRun = now
2071
2072 c.log.Debug().Msg("config hot reload complete")
2073 return nil
2074}
2075
2076// buildProxyConfig parses size-related config fields and builds the proxy config.
2077func buildProxyConfig(cfg Config, log zerowrap.Logger) (*proxyConfigResult, error) {
2078 maxProxyBodySize := int64(512 << 20) // 512MB default
2079 if cfg.Server.MaxProxyBodySize != "" {
2080 parsedSize, err := bytesize.Parse(cfg.Server.MaxProxyBodySize)
2081 if err != nil {
2082 return nil, log.WrapErrWithFields(err, "invalid server.max_proxy_body_size configuration", map[string]any{"value": cfg.Server.MaxProxyBodySize})
2083 }
2084 maxProxyBodySize = parsedSize
2085 }
2086
2087 maxBlobChunkSize := int64(registry.DefaultMaxBlobChunkSize)
2088 if cfg.Server.MaxBlobChunkSize != "" {
2089 parsedSize, err := bytesize.Parse(cfg.Server.MaxBlobChunkSize)
2090 if err != nil {
2091 return nil, log.WrapErrWithFields(err, "invalid server.max_blob_chunk_size configuration", map[string]any{"value": cfg.Server.MaxBlobChunkSize})
2092 }
2093 maxBlobChunkSize = parsedSize
2094 }
2095
2096 maxBlobSize := int64(registry.DefaultMaxBlobSize)
2097 if cfg.Server.MaxBlobSize != "" {
2098 parsedSize, err := bytesize.Parse(cfg.Server.MaxBlobSize)
2099 if err != nil {
2100 return nil, log.WrapErrWithFields(err, "invalid server.max_blob_size configuration", map[string]any{"value": cfg.Server.MaxBlobSize})
2101 }
2102 maxBlobSize = parsedSize
2103 }
2104
2105 maxProxyResponseSize := int64(1 << 30) // 1GB default
2106 if cfg.Server.MaxProxyResponseSize != "" {
2107 parsedSize, err := bytesize.Parse(cfg.Server.MaxProxyResponseSize)
2108 if err != nil {
2109 return nil, log.WrapErrWithFields(err, "invalid server.max_proxy_response_size configuration", map[string]any{"value": cfg.Server.MaxProxyResponseSize})
2110 }
2111 maxProxyResponseSize = parsedSize
2112 }
2113
2114 maxConcurrentConns := cfg.Server.MaxConcurrentConns
2115 if maxConcurrentConns < 0 {
2116 maxConcurrentConns = 10000 // default when explicitly set to -1
2117 }
2118 // 0 means no limit (as documented in proxy.Config)
2119
2120 registryDomain, _ := resolveRegistryDomains(cfg)
2121
2122 return &proxyConfigResult{
2123 proxyConfig: proxy.Config{
2124 RegistryDomain: registryDomain,
2125 RegistryPort: cfg.Server.RegistryPort,
2126 MaxBodySize: maxProxyBodySize,
2127 MaxResponseSize: maxProxyResponseSize,
2128 MaxConcurrentConns: maxConcurrentConns,
2129 },
2130 maxBlobChunkSize: maxBlobChunkSize,
2131 maxBlobSize: maxBlobSize,
2132 }, nil
2133}
2134
2135// buildDNSConfig parses the raw dns config section into a publictls.DNSConfig.
2136func buildDNSConfig(cfg Config) (publictls.DNSConfig, error) {
2137 defaults := publictls.DefaultDNSConfig()
2138
2139 resolvers := cfg.DNS.Resolvers
2140 if len(resolvers) == 0 {
2141 resolvers = defaults.Resolvers
2142 }
2143
2144 propagationTimeout := defaults.PropagationTimeout
2145 if cfg.DNS.PropagationTimeout != "" {
2146 parsed, err := time.ParseDuration(cfg.DNS.PropagationTimeout)
2147 if err != nil {
2148 return publictls.DNSConfig{}, fmt.Errorf("invalid dns.propagation_timeout: %w", err)
2149 }
2150 propagationTimeout = parsed
2151 }
2152
2153 pollingInterval := defaults.PollingInterval
2154 if cfg.DNS.PollingInterval != "" {
2155 parsed, err := time.ParseDuration(cfg.DNS.PollingInterval)
2156 if err != nil {
2157 return publictls.DNSConfig{}, fmt.Errorf("invalid dns.polling_interval: %w", err)
2158 }
2159 pollingInterval = parsed
2160 }
2161
2162 dnsCfg := publictls.DNSConfig{
2163 Resolvers: append([]string(nil), resolvers...),
2164 PropagationTimeout: propagationTimeout,
2165 PollingInterval: pollingInterval,
2166 }
2167 if err := dnsCfg.Validate(); err != nil {
2168 return publictls.DNSConfig{}, err
2169 }
2170 return dnsCfg, nil
2171}
2172
2173func buildContainerServiceConfig(_ context.Context, v *viper.Viper, _ Config, _ *services, _ zerowrap.Logger) (container.Config, error) {
2174 return container.Config{
2175 NetworkPrefix: v.GetString("network_isolation.network_prefix"),
2176 }, nil
2177}
2178
2179// createContainerService creates the container service with configuration.
2180func createContainerService(ctx context.Context, v *viper.Viper, cfg Config, svc *services, log zerowrap.Logger) (*container.Service, error) {
2181 containerConfig, err := buildContainerServiceConfig(ctx, v, cfg, svc, log)
2182 if err != nil {
2183 return nil, err
2184 }
2185 return container.NewService(svc.runtime, svc.eventBus, svc.logWriter, containerConfig), nil
2186}
2187
2188type databaseBackupSettingsConfig struct {
2189 Enabled bool
2190 Schedule string
2191 StorageDir string
2192 Retention struct {
2193 Hourly int
2194 Daily int
2195 Weekly int
2196 Monthly int
2197 }
2198}
2199
2200func databaseBackupSettings(cfg Config) databaseBackupSettingsConfig {
2201 out := databaseBackupSettingsConfig{
2202 Enabled: cfg.Backups.Databases.Enabled,
2203 Schedule: cfg.Backups.Databases.Schedule,
2204 StorageDir: cfg.Backups.Databases.StorageDir,
2205 }
2206 out.Retention.Hourly = cfg.Backups.Databases.Retention.Hourly
2207 out.Retention.Daily = cfg.Backups.Databases.Retention.Daily
2208 out.Retention.Weekly = cfg.Backups.Databases.Retention.Weekly
2209 out.Retention.Monthly = cfg.Backups.Databases.Retention.Monthly
2210 // Legacy [backups] keys intentionally override new database defaults when
2211 // backups.enabled is true, preserving existing working pg_dump schedules.
2212 // Otherwise, prefer the already-populated backups.databases.* values.
2213 if cfg.Backups.Enabled {
2214 out.Enabled = true
2215 if cfg.Backups.Schedule != "" {
2216 out.Schedule = cfg.Backups.Schedule
2217 }
2218 if cfg.Backups.StorageDir != "" {
2219 out.StorageDir = cfg.Backups.StorageDir
2220 }
2221 } else {
2222 if out.Schedule == "" {
2223 out.Schedule = cfg.Backups.Schedule
2224 }
2225 if out.StorageDir == "" {
2226 out.StorageDir = cfg.Backups.StorageDir
2227 }
2228 }
2229 if out.Retention.Hourly == 0 {
2230 out.Retention.Hourly = cfg.Backups.Retention.Hourly
2231 }
2232 if out.Retention.Daily == 0 {
2233 out.Retention.Daily = cfg.Backups.Retention.Daily
2234 }
2235 if out.Retention.Weekly == 0 {
2236 out.Retention.Weekly = cfg.Backups.Retention.Weekly
2237 }
2238 if out.Retention.Monthly == 0 {
2239 out.Retention.Monthly = cfg.Backups.Retention.Monthly
2240 }
2241 return out
2242}
2243
2244func createBackupService(cfg Config, svc *services, log zerowrap.Logger) (*filesystem.BackupStorage, *backup.Service, error) {
2245 dbCfg := databaseBackupSettings(cfg)
2246 if !dbCfg.Enabled {
2247 return nil, nil, nil
2248 }
2249
2250 storageDir := dbCfg.StorageDir
2251 if storageDir == "" {
2252 dataDir := resolveDataDir(cfg.Server.DataDir)
2253 storageDir = filepath.Join(dataDir, "backups")
2254 }
2255
2256 backupStorage, err := filesystem.NewBackupStorage(storageDir, log)
2257 if err != nil {
2258 return nil, nil, log.WrapErr(err, "failed to create backup storage")
2259 }
2260
2261 retention, err := validateBackupRetention(cfg)
2262 if err != nil {
2263 return nil, nil, log.WrapErr(err, "invalid backup retention policy")
2264 }
2265
2266 backupCfg := domain.BackupConfig{
2267 Enabled: dbCfg.Enabled,
2268 StorageDir: storageDir,
2269 Retention: retention,
2270 }
2271
2272 backupSvc := backup.NewService(svc.runtime, backupStorage, backupCfg, log)
2273
2274 log.Info().
2275 Str("storage_dir", storageDir).
2276 Msg("backup service initialized")
2277
2278 return backupStorage, backupSvc, nil
2279}
2280
2281func createVolumeBackupService(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger) (out.VolumeBackupStorage, *backup.VolumeService, domain.VolumeBackupConfig, error) {
2282 if !cfg.Backups.Volumes.Enabled {
2283 return nil, nil, domain.VolumeBackupConfig{}, nil
2284 }
2285 volumeCfg, err := validateVolumeBackupConfig(cfg)
2286 if err != nil {
2287 return nil, nil, domain.VolumeBackupConfig{}, log.WrapErr(err, "invalid volume backup configuration")
2288 }
2289
2290 storage, err := s3storage.NewVolumeBackupStorage(ctx, volumeCfg)
2291 if err != nil {
2292 return nil, nil, domain.VolumeBackupConfig{}, log.WrapErr(err, "failed to create volume backup storage")
2293 }
2294
2295 volumeSvc := backup.NewVolumeService(svc.runtime, storage, volumeCfg, log)
2296 log.Info().
2297 Str("bucket", volumeCfg.S3Bucket).
2298 Str("prefix", volumeCfg.S3Prefix).
2299 Msg("volume backup service initialized")
2300
2301 return storage, volumeSvc, volumeCfg, nil
2302}
2303
2304func validateBackupRetention(cfg Config) (domain.RetentionPolicy, error) {
2305 dbCfg := databaseBackupSettings(cfg)
2306 if dbCfg.Retention.Hourly < 0 {
2307 return domain.RetentionPolicy{}, fmt.Errorf("backups.databases.retention.hourly cannot be negative")
2308 }
2309 if dbCfg.Retention.Daily < 0 {
2310 return domain.RetentionPolicy{}, fmt.Errorf("backups.databases.retention.daily cannot be negative")
2311 }
2312 if dbCfg.Retention.Weekly < 0 {
2313 return domain.RetentionPolicy{}, fmt.Errorf("backups.databases.retention.weekly cannot be negative")
2314 }
2315 if dbCfg.Retention.Monthly < 0 {
2316 return domain.RetentionPolicy{}, fmt.Errorf("backups.databases.retention.monthly cannot be negative")
2317 }
2318
2319 return domain.RetentionPolicy{
2320 Hourly: dbCfg.Retention.Hourly,
2321 Daily: dbCfg.Retention.Daily,
2322 Weekly: dbCfg.Retention.Weekly,
2323 Monthly: dbCfg.Retention.Monthly,
2324 }, nil
2325}
2326
2327func validateVolumeBackupConfig(cfg Config) (domain.VolumeBackupConfig, error) {
2328 volumeCfg := cfg.Backups.Volumes
2329 interval, err := parsePositiveDurationDefault(volumeCfg.Interval, "24h", "backups.volumes.interval")
2330 if err != nil {
2331 return domain.VolumeBackupConfig{}, err
2332 }
2333 timeout, err := parsePositiveDurationDefault(volumeCfg.Timeout, "2h", "backups.volumes.timeout")
2334 if err != nil {
2335 return domain.VolumeBackupConfig{}, err
2336 }
2337 compression, err := parseVolumeBackupCompression(volumeCfg.Compression)
2338 if err != nil {
2339 return domain.VolumeBackupConfig{}, err
2340 }
2341 maxConcurrency := volumeCfg.MaxConcurrency
2342 if maxConcurrency == 0 {
2343 maxConcurrency = 2
2344 }
2345 helperImage := strings.TrimSpace(volumeCfg.HelperImage)
2346 if helperImage == "" {
2347 helperImage = "alpine:3.20"
2348 }
2349 volumePrefix := strings.TrimSpace(cfg.Volumes.Prefix)
2350 if volumePrefix == "" {
2351 volumePrefix = "gordon"
2352 }
2353 if compression == domain.VolumeBackupCompressionZstd && helperImage == "alpine:3.20" {
2354 return domain.VolumeBackupConfig{}, fmt.Errorf("backups.volumes.compression zstd requires a helper_image that provides zstd")
2355 }
2356 if err := validateVolumeBackupS3Settings(volumeCfg.Enabled, volumeCfg.Retention.Keep, maxConcurrency, volumeCfg.S3.Bucket, volumeCfg.S3.Region); err != nil {
2357 return domain.VolumeBackupConfig{}, err
2358 }
2359 return domain.VolumeBackupConfig{
2360 Enabled: volumeCfg.Enabled,
2361 Interval: interval,
2362 Compression: compression,
2363 Retention: domain.VolumeBackupRetentionPolicy{Keep: volumeCfg.Retention.Keep},
2364 Timeout: timeout,
2365 MaxConcurrency: maxConcurrency,
2366 HelperImage: helperImage,
2367 VolumePrefix: volumePrefix,
2368 S3Bucket: strings.TrimSpace(volumeCfg.S3.Bucket),
2369 S3Region: strings.TrimSpace(volumeCfg.S3.Region),
2370 S3Prefix: strings.TrimSpace(volumeCfg.S3.Prefix),
2371 S3Endpoint: strings.TrimSpace(volumeCfg.S3.Endpoint),
2372 S3PathStyle: volumeCfg.S3.PathStyle,
2373 S3SSEAlgorithm: strings.TrimSpace(volumeCfg.S3.SSEAlgorithm),
2374 S3SSEKMSKeyID: strings.TrimSpace(volumeCfg.S3.SSEKMSKeyID),
2375 }, nil
2376}
2377
2378func parsePositiveDurationDefault(raw, defaultValue, field string) (time.Duration, error) {
2379 if strings.TrimSpace(raw) == "" {
2380 raw = defaultValue
2381 }
2382 d, err := time.ParseDuration(strings.TrimSpace(raw))
2383 if err != nil || d <= 0 {
2384 return 0, fmt.Errorf("%s must be a positive duration", field)
2385 }
2386 return d, nil
2387}
2388
2389func parseVolumeBackupCompression(raw string) (domain.VolumeBackupCompression, error) {
2390 if strings.TrimSpace(raw) == "" {
2391 raw = string(domain.VolumeBackupCompressionGzip)
2392 }
2393 compression := domain.VolumeBackupCompression(strings.ToLower(strings.TrimSpace(raw)))
2394 switch compression {
2395 case domain.VolumeBackupCompressionGzip, domain.VolumeBackupCompressionZstd:
2396 return compression, nil
2397 default:
2398 return "", fmt.Errorf("backups.volumes.compression must be one of: gzip, zstd")
2399 }
2400}
2401
2402func validateVolumeBackupS3Settings(enabled bool, keep, maxConcurrency int, bucket, region string) error {
2403 if keep < 0 {
2404 return fmt.Errorf("backups.volumes.retention.keep cannot be negative")
2405 }
2406 if enabled && keep == 0 {
2407 return fmt.Errorf("backups.volumes.retention.keep must be positive when volume backups are enabled")
2408 }
2409 if maxConcurrency < 1 {
2410 return fmt.Errorf("backups.volumes.max_concurrency must be at least 1")
2411 }
2412 if enabled && strings.TrimSpace(bucket) == "" {
2413 return fmt.Errorf("backups.volumes.s3.bucket is required when volume backups are enabled")
2414 }
2415 if enabled && strings.TrimSpace(region) == "" {
2416 return fmt.Errorf("backups.volumes.s3.region is required when volume backups are enabled")
2417 }
2418 return nil
2419}
2420
2421// registerEventHandlers registers event handlers. Push-triggered
2422// route creation (auto-route, previews, image-pushed deploy) is
2423// retired: pushes transfer OCI content only and apps deploy
2424// explicitly via `gordon apps deploy`. The pre-v3 route-engine
2425// deploy arms (config-reload redeploy, manual SIGUSR2 deploy,
2426// secrets-changed redeploy) are removed with the declarative-apps
2427// cutover: reload never activates app state, and app secret changes
2428// take effect on explicit deploy.
2429func registerEventHandlers(ctx context.Context, svc *services) (func(), error) {
2430 // Proxy cache invalidation on config reload (clears stale targets for removed routes)
2431 configReloadProxyHandler := proxy.NewConfigReloadProxyHandler(ctx, svc.proxySvc)
2432 if err := svc.eventBus.Subscribe(configReloadProxyHandler); err != nil {
2433 return nil, fmt.Errorf("failed to subscribe config reload proxy handler: %w", err)
2434 }
2435
2436 cleanup := func() {
2437 if svc.reloadCoordinator != nil {
2438 svc.reloadCoordinator.Stop()
2439 }
2440 }
2441
2442 return cleanup, nil
2443}
2444
2445// setupConfigHotReload sets up config hot reload.
2446func setupConfigHotReload(ctx context.Context, configSvc configWatcher, coordinator loadedConfigApplier) error {
2447 if err := configSvc.Watch(ctx, func() {
2448 _ = coordinator.ApplyLoadedConfig(ctx)
2449 }); err != nil {
2450 return fmt.Errorf("failed to watch config: %w", err)
2451 }
2452
2453 return nil
2454}
2455
2456func loopbackOnly(next http.Handler, log zerowrap.Logger) http.Handler {
2457 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2458 host, _, err := net.SplitHostPort(r.RemoteAddr)
2459 if err != nil {
2460 host = r.RemoteAddr
2461 }
2462
2463 ip := net.ParseIP(host)
2464 if ip == nil || !ip.IsLoopback() {
2465 log.Warn().
2466 Str("path", r.URL.Path).
2467 Str("remote_addr", r.RemoteAddr).
2468 Msg("blocked non-loopback access on internal admin route")
2469 w.Header().Set("Content-Type", "application/json")
2470 w.WriteHeader(http.StatusForbidden)
2471 _ = json.NewEncoder(w).Encode(dto.ErrorResponse{Error: "Forbidden"})
2472 return
2473 }
2474
2475 next.ServeHTTP(w, r)
2476 })
2477}
2478
2479// hostRedirectEligible reports whether a known host should be redirected
2480// from plaintext to HTTPS: app hosts declared tls=never stay plain HTTP.
2481func hostRedirectEligible(svc *services, host string) bool {
2482 if svc.appHostIndex == nil {
2483 return true
2484 }
2485 entry, ok := svc.appHostIndex.Lookup(host)
2486 return !ok || entry.TLSMode != domain.AppTLSNever
2487}
2488
2489// hostRequiresTLS reports whether an app host declared tls=always, which
2490// must never be served over plaintext. An unwired index has no always hosts.
2491func hostRequiresTLS(svc *services, host string) bool {
2492 if svc.appHostIndex == nil {
2493 return false
2494 }
2495 entry, ok := svc.appHostIndex.Lookup(host)
2496 return ok && entry.TLSMode == domain.AppTLSAlways
2497}
2498
2499// createHTTPHandlers creates HTTP handlers with middleware.
2500// Returns three handlers: registry, HTTP proxy (with CIDR + onboarding), and HTTPS proxy.
2501func createHTTPHandlers(svc *services, cfg Config, log zerowrap.Logger, accessWriter out.AccessLogWriter) (http.Handler, http.Handler, http.Handler) {
2502 // Parse trusted proxies once for all middleware chains.
2503 // This ensures consistent IP extraction across logging, rate limiting, and auth.
2504 trustedNets := httphelper.ParseTrustedProxies(cfg.API.RateLimit.TrustedProxies)
2505
2506 // Registry handler
2507 registryHandler := registry.NewHandler(svc.registrySvc, log, svc.maxBlobChunkSize, svc.maxBlobSize)
2508 svc.registryHandler = registryHandler
2509 if svc.reloadCoordinator != nil {
2510 svc.reloadCoordinator.SetRegistryLimits(registryHandler)
2511 }
2512 registryWithMiddleware, cidrAllowlistMiddleware, rateLimitMiddleware := buildRegistryHandlerWithMiddleware(
2513 svc,
2514 cfg,
2515 trustedNets,
2516 registryHandler,
2517 log,
2518 )
2519
2520 registryMux := http.NewServeMux()
2521 registerAuthRoutes(registryMux, svc, trustedNets, cidrAllowlistMiddleware, rateLimitMiddleware, cfg, log)
2522 registryMux.Handle("/v2/", wrapRegistryForLocalMode(registryWithMiddleware, cfg, log))
2523 registerAdminRoutes(registryMux, svc, cfg, trustedNets, log)
2524
2525 // Proxy handler. Registry-domain forwarding is gated on auth: with
2526 // auth disabled the registry stays local-only and public ingress can
2527 // never reach it through this proxy.
2528 proxyHandler := proxyadapter.NewHandler(svc.proxySvc, trustedNets, log).
2529 WithRegistryForwarding(cfg.Auth.Enabled)
2530
2531 // HTTP proxy handler chain: HTTPS redirect for non-proxy clients, then CIDR allowlist
2532 proxyAllowedNets, proxyCIDRMiddleware := buildProxyCIDRAllowlistMiddleware(cfg, trustedNets, log)
2533
2534 httpProxyMiddlewares := []func(http.Handler) http.Handler{
2535 middleware.PanicRecovery(log),
2536 middleware.RequestLogger(log, trustedNets),
2537 middleware.SecurityHeaders,
2538 middleware.HTTPSRedirectWithEligibility(proxyAllowedNets, effectiveProxyHTTPPort(cfg), effectiveProxyTLSPort(cfg), cfg.Server.ForceHTTPSRedirect, log, func(host string) bool {
2539 return svc.proxySvc.IsKnownHost(context.Background(), host)
2540 }, func(host string) bool {
2541 return hostRedirectEligible(svc, host)
2542 }, func(host string) bool {
2543 return hostRequiresTLS(svc, host)
2544 }),
2545 }
2546 if proxyCIDRMiddleware != nil {
2547 httpProxyMiddlewares = append(httpProxyMiddlewares, proxyCIDRMiddleware)
2548 }
2549
2550 httpProxyWithMiddleware := otelhttp.NewHandler(
2551 middleware.Chain(httpProxyMiddlewares...)(proxyHandler),
2552 "gordon.proxy",
2553 )
2554
2555 // Build the onboarding handler once if internal CA is available and TLS is enabled.
2556 var obHandler *onboarding.Handler
2557 if svc.caAdapter != nil && effectiveProxyTLSPort(cfg) != 0 {
2558 mobileconfigBytes := pkiadapter.GenerateMobileconfig(
2559 svc.caAdapter.RootCertificateDER(),
2560 svc.caAdapter.RootCommonName(),
2561 )
2562 obHandler = onboarding.NewHandler(
2563 svc.caAdapter.RootCertificate(),
2564 mobileconfigBytes,
2565 svc.caAdapter.RootFingerprint(),
2566 effectiveProxyHTTPPort(cfg),
2567 effectiveProxyTLSPort(cfg),
2568 )
2569 }
2570
2571 // HTTP proxyMux: trusted proxy traffic flows through the normal proxy chain.
2572 // Direct clients get an onboarding gate (when CA is available) placed BEFORE
2573 // HTTPSRedirect so force_https_redirect cannot bypass onboarding.
2574 // ACME HTTP-01 challenge handler is registered before the catch-all "/" so
2575 // it gets first chance regardless of source IP.
2576 proxyMux := http.NewServeMux()
2577
2578 // Register ACME HTTP-01 challenge handler before all other routes so
2579 // Let's Encrypt validation always succeeds, even for onboarding clients.
2580 if svc.publicTLSSvc != nil {
2581 proxyMux.Handle(acmehttp.Prefix, acmehttp.NewHandler(svc.publicTLSSvc))
2582 }
2583
2584 if proxyCIDRMiddleware != nil && proxyAllowedNets == nil {
2585 // Invalid proxy_allowed_ips: deny all traffic (fail-closed).
2586 proxyMux.Handle("/", proxyCIDRMiddleware(httpProxyWithMiddleware))
2587 } else if obHandler != nil {
2588 proxyMux.Handle("/", directHTTPOnboardingGate(obHandler, proxyAllowedNets, httpProxyWithMiddleware, log))
2589 } else {
2590 proxyMux.Handle("/", httpProxyWithMiddleware)
2591 }
2592
2593 // HTTPS proxy handler chain: security headers + proxy + CA onboarding
2594 // Onboarding routes live on the TLS port so Tailnet / direct clients
2595 // can click through the initial cert warning, install the CA, and
2596 // then trust all subsequent connections.
2597 // The middleware chain wraps the entire mux so onboarding routes also
2598 // get PanicRecovery, RequestLogger, and SecurityHeaders.
2599 httpsProxyMiddlewares := []func(http.Handler) http.Handler{
2600 middleware.PanicRecovery(log),
2601 middleware.RequestLogger(log, trustedNets),
2602 middleware.SecurityHeaders,
2603 }
2604
2605 httpsMux := http.NewServeMux()
2606 if obHandler != nil && cfg.Server.GordonDomain != "" {
2607 gordonDomain := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(cfg.Server.GordonDomain)), ".")
2608 onboardingMux := http.NewServeMux()
2609 registerOnboardingRoutes(onboardingMux, obHandler)
2610 // Register onboarding paths host-gated so normal traffic hits
2611 // proxyHandler directly through the catch-all / pattern.
2612 httpsMux.Handle("GET /.well-known/gordon/", gordonDomainOnboardingGate(gordonDomain, onboardingMux, proxyHandler))
2613 httpsMux.Handle("GET /.well-known/gordon/ca", gordonDomainOnboardingGate(gordonDomain, onboardingMux, proxyHandler))
2614 httpsMux.Handle("GET /.well-known/gordon/ca.crt", gordonDomainOnboardingGate(gordonDomain, onboardingMux, proxyHandler))
2615 httpsMux.Handle("GET /.well-known/gordon/ca.mobileconfig", gordonDomainOnboardingGate(gordonDomain, onboardingMux, proxyHandler))
2616 httpsMux.Handle("/", proxyHandler)
2617 } else {
2618 httpsMux.Handle("/", proxyHandler)
2619 }
2620
2621 httpsHandler := otelhttp.NewHandler(middleware.Chain(httpsProxyMiddlewares...)(httpsMux), "gordon.proxy.tls")
2622
2623 // Wrap top-level handlers with access logging outside all gates
2624 // (loopbackOnly, denyAllHandler, CIDR allowlist) so every request —
2625 // including rejected probes — produces exactly one access-log line.
2626 var registryOut, proxyOut, httpsOut http.Handler = registryMux, proxyMux, httpsHandler
2627 if accessWriter != nil {
2628 excludeHC := cfg.Logging.AccessLog.ExcludeHealthChecks
2629 registryOut = middleware.AccessLogger(accessWriter, excludeHC, log, trustedNets)(registryOut)
2630 proxyOut = middleware.AccessLogger(accessWriter, excludeHC, log, trustedNets)(proxyOut)
2631 httpsOut = middleware.AccessLogger(accessWriter, excludeHC, log, trustedNets)(httpsOut)
2632 }
2633 svc.httpsProxyHandler = httpsOut
2634
2635 return registryOut, proxyOut, httpsOut
2636}
2637
2638// registerOnboardingRoutes registers CA onboarding well-known HTTP routes on
2639// the given mux. Both direct-HTTP and Gordon-domain HTTPS onboarding use this.
2640func registerOnboardingRoutes(mux *http.ServeMux, ob *onboarding.Handler) {
2641 mux.HandleFunc("GET /.well-known/gordon/", ob.ServeOnboardingPage)
2642 mux.HandleFunc("GET /.well-known/gordon/ca", ob.ServeOnboardingPage)
2643 mux.HandleFunc("GET /.well-known/gordon/ca.crt", ob.ServeCACert)
2644 mux.HandleFunc("GET /.well-known/gordon/ca.mobileconfig", ob.ServeMobileconfig)
2645}
2646
2647// directHTTPOnboardingGate returns an http.Handler that splits HTTP traffic
2648// by source IP. Trusted proxy IPs flow through to the normal proxy chain.
2649// Direct clients are served the CA onboarding flow on allowed paths and
2650// receive 403 on everything else. This gate runs BEFORE HTTPSRedirect so
2651// force_https_redirect cannot bypass onboarding for direct clients.
2652func directHTTPOnboardingGate(ob *onboarding.Handler, proxyNets []*net.IPNet, proxyChain http.Handler, log zerowrap.Logger) http.Handler {
2653 // Build a small mux for direct-client onboarding paths.
2654 onboardingMux := http.NewServeMux()
2655 registerOnboardingRoutes(onboardingMux, ob)
2656
2657 // Reserve ACME challenge path for future use.
2658 onboardingMux.HandleFunc("/.well-known/acme-challenge/", func(w http.ResponseWriter, _ *http.Request) {
2659 http.Error(w, "not found", http.StatusNotFound)
2660 })
2661
2662 // Catch-all: reject any other direct HTTP request.
2663 // Uses a method-aware split: GET writes a body, HEAD gets an empty 403.
2664 onboardingMux.HandleFunc("/", directHTTPForbidden)
2665
2666 onboardingWithMiddleware := middleware.Chain(
2667 middleware.PanicRecovery(log),
2668 middleware.RequestLogger(log), // Intentionally omit trusted proxy nets so direct onboarding logs use RemoteAddr only.
2669 middleware.SecurityHeaders,
2670 )(onboardingMux)
2671
2672 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2673 remoteIP := httphelper.ExtractRemoteIP(r.RemoteAddr)
2674 if httphelper.IsTrustedOrLocal(remoteIP, proxyNets) {
2675 proxyChain.ServeHTTP(w, r)
2676 return
2677 }
2678 onboardingWithMiddleware.ServeHTTP(w, r)
2679 })
2680}
2681
2682// canonicalHostsEqual compares two hosts after normalising both: stripping
2683// port, trimming spaces, lowercasing, and removing trailing dot.
2684func canonicalHostsEqual(host, expected string) bool {
2685 host = strings.TrimSpace(host)
2686 if h, _, err := net.SplitHostPort(host); err == nil {
2687 host = h
2688 }
2689 host = strings.TrimSuffix(strings.ToLower(host), ".")
2690 expected = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(expected)), ".")
2691 return host == expected
2692}
2693
2694// gordonDomainOnboardingGate returns a handler that serves onboarding routes
2695// only when the request host matches gordonDomain. For mismatched hosts it
2696// delegates to proxyHandler. gordonDomain must already be canonicalised
2697// (trimmed, lowered, trailing dot removed).
2698func gordonDomainOnboardingGate(gordonDomain string, onboardingMux, proxyHandler http.Handler) http.Handler {
2699 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2700 if gordonDomain != "" && canonicalHostsEqual(r.Host, gordonDomain) {
2701 onboardingMux.ServeHTTP(w, r)
2702 return
2703 }
2704 proxyHandler.ServeHTTP(w, r)
2705 })
2706}
2707
2708// directHTTPForbidden responds with 403 for non-onboarding HTTP paths.
2709// HEAD requests get an empty body per HTTP semantics.
2710func directHTTPForbidden(w http.ResponseWriter, r *http.Request) {
2711 w.Header().Set("Content-Type", "text/plain; charset=utf-8")
2712 w.WriteHeader(http.StatusForbidden)
2713 if r.Method != http.MethodHead {
2714 _, _ = w.Write([]byte("Only certificate onboarding is available over HTTP.\n"))
2715 }
2716}
2717
2718func buildRegistryHandlerWithMiddleware(
2719 svc *services,
2720 cfg Config,
2721 trustedNets []*net.IPNet,
2722 registryHandler http.Handler,
2723 log zerowrap.Logger,
2724) (http.Handler, func(http.Handler) http.Handler, func(http.Handler) http.Handler) {
2725 registryMiddlewares := []func(http.Handler) http.Handler{
2726 middleware.PanicRecovery(log),
2727 middleware.RequestLogger(log, trustedNets),
2728 middleware.SecurityHeaders,
2729 }
2730
2731 cidrAllowlistMiddleware := buildRegistryCIDRAllowlistMiddleware(cfg, trustedNets, log)
2732 if cidrAllowlistMiddleware != nil {
2733 registryMiddlewares = append(registryMiddlewares, cidrAllowlistMiddleware)
2734 }
2735
2736 rateLimitMiddleware := buildRegistryRateLimitMiddleware(cfg, log)
2737 registryMiddlewares = append(registryMiddlewares, rateLimitMiddleware)
2738
2739 appendRegistryAuthMiddleware(®istryMiddlewares, svc, cfg, trustedNets, log)
2740
2741 registryWithOtel := otelhttp.NewHandler(
2742 middleware.Chain(registryMiddlewares...)(registryHandler),
2743 "gordon.registry",
2744 )
2745 return registryWithOtel, cidrAllowlistMiddleware, rateLimitMiddleware
2746}
2747
2748// parseCIDRAllowlist parses a list of IPs/CIDRs, logs warnings for invalid entries,
2749// and returns the parsed nets. label is used in log messages (e.g. "registry_allowed_ips").
2750func parseCIDRAllowlist(ips []string, label string, log zerowrap.Logger) ([]*net.IPNet, bool) {
2751 if len(ips) == 0 {
2752 return nil, false
2753 }
2754
2755 allowedNets := httphelper.ParseTrustedProxies(ips)
2756 if len(allowedNets) != len(ips) {
2757 for _, entry := range ips {
2758 if nets := httphelper.ParseTrustedProxies([]string{entry}); len(nets) == 0 {
2759 log.Warn().Str("entry", entry).Msgf("ignoring invalid %s entry", label)
2760 }
2761 }
2762 }
2763
2764 if len(allowedNets) == 0 {
2765 log.Error().
2766 Strs(label, ips).
2767 Msgf("%s is set but no valid entries were parsed; will deny all traffic (fail-closed)", label)
2768 return nil, true // allInvalid
2769 }
2770
2771 return allowedNets, false
2772}
2773
2774// denyAllHandler returns a middleware that rejects every request with 403 Forbidden.
2775func denyAllHandler(label string, trustedNets []*net.IPNet, log zerowrap.Logger) func(http.Handler) http.Handler {
2776 return func(next http.Handler) http.Handler {
2777 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2778 log.Warn().
2779 Str(zerowrap.FieldClientIP, middleware.GetClientIP(r, trustedNets)).
2780 Msgf("access denied due to invalid %s configuration", label)
2781 w.Header().Set("Content-Type", "application/json")
2782 w.WriteHeader(http.StatusForbidden)
2783 _ = json.NewEncoder(w).Encode(dto.ErrorResponse{Error: "Forbidden"})
2784 })
2785 }
2786}
2787
2788func buildRegistryCIDRAllowlistMiddleware(cfg Config, trustedNets []*net.IPNet, log zerowrap.Logger) func(http.Handler) http.Handler {
2789 allowedNets, allInvalid := parseCIDRAllowlist(cfg.Server.RegistryAllowedIPs, "registry_allowed_ips", log)
2790 if allInvalid {
2791 return denyAllHandler("registry_allowed_ips", trustedNets, log)
2792 }
2793 if allowedNets == nil {
2794 return nil
2795 }
2796 return middleware.RegistryCIDRAllowlist(allowedNets, trustedNets, log)
2797}
2798
2799func buildProxyCIDRAllowlistMiddleware(cfg Config, trustedNets []*net.IPNet, log zerowrap.Logger) ([]*net.IPNet, func(http.Handler) http.Handler) {
2800 allowedNets, allInvalid := parseCIDRAllowlist(cfg.Server.ProxyAllowedIPs, "proxy_allowed_ips", log)
2801 if allInvalid {
2802 return nil, denyAllHandler("proxy_allowed_ips", trustedNets, log)
2803 }
2804 if allowedNets == nil {
2805 return nil, nil
2806 }
2807
2808 log.Info().
2809 Strs("proxy_allowed_ips", cfg.Server.ProxyAllowedIPs).
2810 Msg("proxy origin IP allowlist enabled")
2811
2812 return allowedNets, middleware.ProxyCIDRAllowlist(allowedNets, log)
2813}
2814
2815func buildRegistryRateLimitMiddleware(cfg Config, log zerowrap.Logger) func(http.Handler) http.Handler {
2816 if cfg.API.RateLimit.Enabled {
2817 globalLimiter := ratelimit.NewMemoryStore(cfg.API.RateLimit.GlobalRPS, cfg.API.RateLimit.Burst, log)
2818 ipLimiter := ratelimit.NewMemoryStore(cfg.API.RateLimit.PerIPRPS, cfg.API.RateLimit.Burst, log)
2819 return registry.RateLimitMiddleware(
2820 globalLimiter,
2821 ipLimiter,
2822 cfg.API.RateLimit.TrustedProxies,
2823 log,
2824 )
2825 }
2826
2827 return registry.RateLimitMiddleware(nil, nil, nil, log)
2828}
2829
2830func appendRegistryAuthMiddleware(registryMiddlewares *[]func(http.Handler) http.Handler, svc *services, cfg Config, trustedNets []*net.IPNet, log zerowrap.Logger) {
2831 if svc.authSvc != nil {
2832 internalAuth := middleware.InternalRegistryAuth{
2833 Username: svc.internalRegUser,
2834 Password: svc.internalRegPass,
2835 }
2836 *registryMiddlewares = append(*registryMiddlewares, middleware.RegistryAuthV2(svc.authSvc, internalAuth, trustedNets, log))
2837 return
2838 }
2839
2840 if cfg.Auth.Enabled {
2841 log.Error().Msg("authentication service unavailable; registry requests will be denied")
2842 *registryMiddlewares = append(*registryMiddlewares, func(next http.Handler) http.Handler {
2843 return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
2844 w.Header().Set("Content-Type", "application/json")
2845 w.WriteHeader(http.StatusServiceUnavailable)
2846 _ = json.NewEncoder(w).Encode(dto.ErrorResponse{Error: "authentication service unavailable"})
2847 })
2848 })
2849 }
2850}
2851
2852func registerAuthRoutes(
2853 registryMux *http.ServeMux,
2854 svc *services,
2855 trustedNets []*net.IPNet,
2856 cidrAllowlistMiddleware func(http.Handler) http.Handler,
2857 rateLimitMiddleware func(http.Handler) http.Handler,
2858 cfg Config,
2859 log zerowrap.Logger,
2860) {
2861 if svc.authHandler == nil {
2862 return
2863 }
2864
2865 // Auth endpoints always get rate limiting, even if global rate limiting is disabled.
2866 // This prevents brute-force attacks against password/token endpoints.
2867 authRateLimitMiddleware := rateLimitMiddleware
2868 if !cfg.API.RateLimit.Enabled {
2869 authGlobalLimiter := ratelimit.NewMemoryStore(50, 100, log)
2870 authIPLimiter := ratelimit.NewMemoryStore(5, 10, log)
2871 authRateLimitMiddleware = registry.RateLimitMiddleware(authGlobalLimiter, authIPLimiter, cfg.API.RateLimit.TrustedProxies, log)
2872 }
2873
2874 // Auth endpoints are NOT protected by auth - they're where clients authenticate
2875 // but still need rate limiting to prevent brute force attacks.
2876 authMiddlewares := []func(http.Handler) http.Handler{
2877 middleware.PanicRecovery(log),
2878 middleware.RequestLogger(log, trustedNets),
2879 middleware.SecurityHeaders,
2880 }
2881 if cidrAllowlistMiddleware != nil {
2882 authMiddlewares = append(authMiddlewares, cidrAllowlistMiddleware)
2883 }
2884 authMiddlewares = append(authMiddlewares, authRateLimitMiddleware)
2885 authWithMiddleware := otelhttp.NewHandler(
2886 middleware.Chain(authMiddlewares...)(svc.authHandler),
2887 "gordon.auth",
2888 )
2889 registryMux.Handle("/auth/", authWithMiddleware)
2890}
2891
2892func wrapRegistryForLocalMode(registryWithMiddleware http.Handler, cfg Config, log zerowrap.Logger) http.Handler {
2893 if !cfg.Auth.Enabled {
2894 return loopbackOnly(registryWithMiddleware, log)
2895 }
2896 return registryWithMiddleware
2897}
2898
2899func registerAdminRoutes(registryMux *http.ServeMux, svc *services, cfg Config, trustedNets []*net.IPNet, log zerowrap.Logger) {
2900 if svc.adminHandler == nil {
2901 return
2902 }
2903
2904 if !cfg.Auth.Enabled {
2905 log.Warn().Msg("auth disabled: admin API endpoints are not registered")
2906 return
2907 }
2908
2909 adminMiddlewares := []func(http.Handler) http.Handler{
2910 middleware.PanicRecovery(log),
2911 middleware.RequestLogger(log, trustedNets),
2912 middleware.SecurityHeaders,
2913 }
2914
2915 if svc.authSvc != nil {
2916 // Create rate limiters for admin API - uses same config as registry.
2917 var globalLimiter, ipLimiter out.RateLimiter
2918 if cfg.API.RateLimit.Enabled {
2919 globalLimiter = ratelimit.NewMemoryStore(cfg.API.RateLimit.GlobalRPS, cfg.API.RateLimit.Burst, log)
2920 ipLimiter = ratelimit.NewMemoryStore(cfg.API.RateLimit.PerIPRPS, cfg.API.RateLimit.Burst, log)
2921 }
2922 adminMiddlewares = append(adminMiddlewares, admin.AuthMiddleware(svc.authSvc, globalLimiter, ipLimiter, trustedNets, log))
2923 } else {
2924 adminMiddlewares = append(adminMiddlewares, func(next http.Handler) http.Handler {
2925 return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
2926 w.Header().Set("Content-Type", "application/json")
2927 w.WriteHeader(http.StatusServiceUnavailable)
2928 _ = json.NewEncoder(w).Encode(dto.ErrorResponse{Error: "authentication service unavailable"})
2929 })
2930 })
2931 }
2932
2933 adminWithMiddleware := otelhttp.NewHandler(
2934 middleware.Chain(adminMiddlewares...)(svc.adminHandler),
2935 "gordon.admin",
2936 )
2937 registryMux.Handle("/admin/", adminWithMiddleware)
2938}
2939
2940// runServers starts the HTTP servers and waits for shutdown.
2941// Signal handling notes:
2942// - SIGINT/SIGTERM: Triggers graceful shutdown via signal.NotifyContext
2943// - SIGUSR1: Triggers config reload without restart
2944// (SIGUSR2 manual route deploy was removed with the declarative-apps
2945// cutover; app deploys are explicit via `gordon apps deploy`.)
2946// The deferred signal.Stop calls ensure signal handlers are properly
2947// cleaned up before program exit, preventing signal handler leaks.
2948func runServers(ctx context.Context, v *viper.Viper, cfg Config, svc *services, reload reloadTrigger, cleanupHandlers func(), log zerowrap.Logger) error {
2949 // Initialize access log writer. Kept here (not in Run) to keep Run's cyclomatic
2950 // complexity within the project limit of 15.
2951 accessWriterConcrete, err := initAccessLog(cfg, log)
2952 if err != nil {
2953 return err
2954 }
2955 if accessWriterConcrete != nil {
2956 defer accessWriterConcrete.Close()
2957 }
2958 // Convert to interface only when non-nil to avoid the Go nil-interface pitfall
2959 // where a typed nil pointer becomes a non-nil interface value.
2960 var accessWriter out.AccessLogWriter
2961 if accessWriterConcrete != nil {
2962 accessWriter = accessWriterConcrete
2963 }
2964 accessWriter = withAccessLogExport(svc, accessWriter)
2965 ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
2966 defer cancel()
2967
2968 // Set up SIGUSR1 for reload.
2969 // Note: signal.Stop must be called (via defer) to release the channel
2970 // and prevent signal handler leaks when the function returns.
2971 reloadChan := make(chan os.Signal, 1)
2972 signal.Notify(reloadChan, syscall.SIGUSR1)
2973 defer signal.Stop(reloadChan)
2974
2975 errChan := make(chan error, 4)
2976
2977 registryHandler, httpProxyHandler, httpsProxyHandler := createHTTPHandlers(svc, cfg, log, accessWriter)
2978 svc.httpProxyHandler = httpProxyHandler
2979 svc.httpsProxyHandler = httpsProxyHandler
2980
2981 registrySrv, registryReady, internalRegistrySrv, internalRegistryReady := startRegistryServers(cfg, registryHandler, errChan, log)
2982
2983 // closeStarted shuts down any servers that were started before an error occurred,
2984 // preventing leaked listeners during partial startup failures.
2985 closeStarted := func(servers ...*http.Server) {
2986 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
2987 defer shutdownCancel()
2988 for _, srv := range servers {
2989 if srv != nil {
2990 if err := srv.Shutdown(shutdownCtx); err != nil {
2991 log.Error().Err(err).Msg("failed to shut down server during startup cleanup")
2992 }
2993 }
2994 }
2995 }
2996
2997 proxySrv, proxyReady, tlsSrv, _, err := startProxyServers(cfg, httpProxyHandler, httpsProxyHandler, svc.pkiSvc, svc.publicTLSSvc, svc.trafficManager, log)
2998 if err != nil {
2999 closeStarted(registrySrv, internalRegistrySrv)
3000 return err
3001 }
3002
3003 // Owner-only local administration socket. Started before readiness is
3004 // announced so the CLI can reach the daemon as soon as it is usable,
3005 // and closed on every exit path. TCP /admin/* registration is unchanged.
3006 localAdmin, err := startLocalAdminServer(svc, errChan, log)
3007 if err != nil {
3008 closeStarted(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3009 shutdownTrafficManagerForStartupCleanup(svc.trafficManager, log)
3010 return err
3011 }
3012
3013 // cleanupAfterLocalAdmin releases the local admin socket and traffic
3014 // manager in addition to the servers that already bound, so no startup
3015 // failure after this point leaks a listening socket or a live manager.
3016 cleanupAfterLocalAdmin := func(servers ...*http.Server) {
3017 cleanupStartupResources(localAdmin, svc.trafficManager, log, servers...)
3018 }
3019
3020 svc.tlsHTTPEntryPoints = tlsMuxHTTPServerNames(cfg)
3021 svc.smartHTTPEntryPoints = smartTCPHTTPServerNames(cfg)
3022
3023 // Wait for both registry listeners and the HTTP proxy before applying traffic.
3024 if err := waitForServerReady(ctx, internalRegistryReady, errChan); err != nil {
3025 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3026 return err
3027 }
3028 if err := waitForCoreProxyReady(ctx, registryReady, proxyReady, errChan); err != nil {
3029 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3030 return err
3031 }
3032 // Apply only the installation graph fail-fast before slower app
3033 // readiness checks. Persisted ACTIVE binds are intentionally excluded
3034 // until boot reconciliation re-inspects them.
3035 if err := applyTrafficRuntimeConfig(ctx, svc.trafficManager, cfg, svc.configSvc, nil); err != nil {
3036 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3037 return err
3038 }
3039
3040 // Reconcile ACTIVE and rebuild its host index before announcing the
3041 // daemon ready. Applying traffic first publishes an empty app graph and
3042 // leaves healthy converged apps returning 404 until a later lifecycle
3043 // operation happens to rebuild it.
3044 reconcileAppsAtBoot(ctx, svc, log)
3045 stopLogExport := startContainerLogExport(ctx, svc, log)
3046 defer stopLogExport()
3047
3048 logEvent := log.Info().
3049 Int("proxy_port", cfg.Server.Port).
3050 Int("registry_port", cfg.Server.RegistryPort)
3051 if cfg.Server.TLSPort != 0 {
3052 logEvent = logEvent.Int("tls_port", cfg.Server.TLSPort)
3053 }
3054 logEvent.Msg("Gordon is running")
3055
3056 startPublicTLSRuntimeWithWarning(ctx, svc.publicTLSRuntime, log)
3057
3058 schedulerCleanup, err := startOptionalSchedulers(ctx, cfg, svc, log, v)
3059 if err != nil {
3060 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3061 return err
3062 }
3063 if schedulerCleanup != nil {
3064 defer schedulerCleanup()
3065 }
3066
3067 waitForShutdown(ctx, errChan, reloadChan, reload, svc.eventBus, log)
3068 cleanupHandlers() // Stop debounce timers before draining containers
3069 // Stop app administration before tearing down traffic/runtime dependencies,
3070 // so no new mutation can begin during shutdown.
3071 localAdmin.Close()
3072 registrySrvs := []*http.Server{registrySrv, internalRegistrySrv}
3073 return gracefulShutdown(registrySrvs, proxySrv, tlsSrv, svc.containerSvc, svc.proxySvc, svc.pkiSvc, svc.publicTLSSvc, svc.trafficManager, svc.appMonitor, svc.appSvcImpl, svc.appState, log)
3074}
3075
3076// reconcileAppsAtBoot reconciles declarative apps intended to run,
3077// rebuilds the proxy host index from reconciled ACTIVE state, then
3078// starts the single periodic recovery monitor. Failures only warn: the
3079// monitor still starts so a partially failed boot self-heals.
3080func reconcileAppsAtBoot(ctx context.Context, svc *services, log zerowrap.Logger) {
3081 if svc.appDeploySvc == nil {
3082 return
3083 }
3084 if err := svc.appDeploySvc.ReconcileBoot(ctx); err != nil {
3085 log.Warn().Err(err).Msg("failed to reconcile apps at boot")
3086 }
3087 if err := rebuildAppHostIndex(ctx, svc); err != nil {
3088 log.Warn().Err(err).Msg("failed to rebuild app host index at boot")
3089 }
3090 // Exactly one monitor per process: a second boot reconciliation must
3091 // not orphan a running loop in the daemon context.
3092 if svc.appMonitor == nil {
3093 svc.appMonitor = newAppMonitor(svc.appDeploySvc, log)
3094 }
3095 svc.appMonitor.Start(ctx)
3096}
3097
3098// workloadLogExporter returns the OTLP workload log exporter, or nil
3099// when telemetry log export is disabled. A typed nil must not leak into
3100// the out.LogExporter interface.
3101func workloadLogExporter(provider *telemetry.Provider) out.LogExporter {
3102 if provider == nil || provider.WorkloadLogs == nil {
3103 return nil
3104 }
3105 return provider.WorkloadLogs
3106}
3107
3108// withAccessLogExport also exports access entries over OTLP when
3109// workload log export is enabled. next may be nil (no local sink).
3110func withAccessLogExport(svc *services, next out.AccessLogWriter) out.AccessLogWriter {
3111 if svc.workloadLogs == nil {
3112 return next
3113 }
3114 return logexport.NewAccessLogExporter(svc.appHostIndex, svc.workloadLogs, next)
3115}
3116
3117// startContainerLogExport follows ACTIVE app containers and exports
3118// their output when telemetry log export is enabled. The returned stop
3119// function cancels every follower and waits for them, so buffered
3120// records reach the exporter before telemetry shuts down.
3121func startContainerLogExport(ctx context.Context, svc *services, log zerowrap.Logger) func() {
3122 if svc.workloadLogs == nil || svc.appState == nil || svc.runtime == nil {
3123 return func() {}
3124 }
3125 ctx, cancel := context.WithCancel(ctx)
3126 done := make(chan struct{})
3127 collector := logexport.NewCollector(svc.appState, svc.runtime, svc.workloadLogs)
3128 go func() {
3129 defer close(done)
3130 collector.Run(ctx)
3131 }()
3132 log.Info().Msg("app container log export enabled")
3133 return func() {
3134 cancel()
3135 <-done
3136 }
3137}
3138
3139func startPublicTLSRuntimeWithWarning(ctx context.Context, svc publicTLSRuntime, log zerowrap.Logger) {
3140 if err := startPublicTLSRuntime(ctx, svc, log); err != nil {
3141 log.Warn().Err(err).Msg("initial public ACME reconcile failed, continuing with renewal loop")
3142 }
3143}
3144
3145func waitForCoreProxyReady(ctx context.Context, registryReady <-chan struct{}, proxyReady <-chan struct{}, errChan <-chan error) error {
3146 if err := waitForServerReady(ctx, registryReady, errChan); err != nil {
3147 return err
3148 }
3149 return waitForServerReady(ctx, proxyReady, errChan)
3150}
3151
3152// shutdownStartedServers gracefully shuts down listeners that bound before a
3153// partial startup failure so an error return does not leak them.
3154func shutdownStartedServers(servers []*http.Server, log zerowrap.Logger) {
3155 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
3156 defer shutdownCancel()
3157 for _, srv := range servers {
3158 if srv != nil {
3159 if err := srv.Shutdown(shutdownCtx); err != nil {
3160 log.Error().Err(err).Msg("failed to shut down server during startup cleanup")
3161 }
3162 }
3163 }
3164}
3165
3166// cleanupStartupResources releases every resource a partial startup may have
3167// acquired once the local admin socket is serving: the admin socket first (no
3168// further mutations can begin), then the bound servers, then the traffic
3169// manager that startProxyServers may already have populated.
3170func cleanupStartupResources(localAdmin *localAdminServer, trafficManager *trafficadapter.Manager, log zerowrap.Logger, servers ...*http.Server) {
3171 if localAdmin != nil {
3172 localAdmin.Close()
3173 }
3174 shutdownStartedServers(servers, log)
3175 shutdownTrafficManagerForStartupCleanup(trafficManager, log)
3176}
3177
3178func shutdownTrafficManagerForStartupCleanup(manager *trafficadapter.Manager, log zerowrap.Logger) {
3179 if manager == nil {
3180 return
3181 }
3182 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
3183 defer shutdownCancel()
3184 if err := manager.Shutdown(shutdownCtx); err != nil {
3185 log.Error().Err(err).Msg("failed to shut down traffic manager during startup cleanup")
3186 }
3187}
3188
3189func waitForServerReady(ctx context.Context, ready <-chan struct{}, errChan <-chan error) error {
3190 if ready == nil {
3191 return nil
3192 }
3193 select {
3194 case <-ready:
3195 return nil
3196 case err := <-errChan:
3197 return err
3198 case <-ctx.Done():
3199 return ctx.Err()
3200 }
3201}
3202
3203func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger, v *viper.Viper) (func(), error) {
3204 schedulers := make([]*cronSvc.Scheduler, 0, 3)
3205
3206 backupScheduler, err := startBackupScheduler(ctx, cfg, svc, log)
3207 if err != nil {
3208 return nil, err
3209 }
3210 if backupScheduler != nil {
3211 schedulers = append(schedulers, backupScheduler)
3212 }
3213
3214 volumeBackupScheduler, err := startVolumeBackupScheduler(ctx, cfg, svc, log)
3215 if err != nil {
3216 return nil, err
3217 }
3218 if volumeBackupScheduler != nil {
3219 schedulers = append(schedulers, volumeBackupScheduler)
3220 }
3221
3222 imageScheduler, err := startImagePruneScheduler(ctx, cfg, svc, log, func() int {
3223 return v.GetInt("images.prune.keep_last")
3224 })
3225 if err != nil {
3226 return nil, err
3227 }
3228 if imageScheduler != nil {
3229 schedulers = append(schedulers, imageScheduler)
3230 }
3231
3232 if len(schedulers) == 0 {
3233 return nil, nil
3234 }
3235
3236 return func() {
3237 for i := len(schedulers) - 1; i >= 0; i-- {
3238 schedulers[i].Stop()
3239 }
3240 }, nil
3241}
3242
3243func startBackupScheduler(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger) (*cronSvc.Scheduler, error) {
3244 dbCfg := databaseBackupSettings(cfg)
3245 if !dbCfg.Enabled || svc == nil || svc.backupSvc == nil {
3246 return nil, nil
3247 }
3248
3249 preset, err := resolveBackupSchedule(dbCfg.Schedule)
3250 if err != nil {
3251 return nil, err
3252 }
3253
3254 scheduler := cronSvc.NewScheduler(log)
3255 err = scheduler.Add(
3256 "backup-scheduler",
3257 "Backups",
3258 domain.CronSchedule{Preset: preset},
3259 func(jobCtx context.Context) error {
3260 if err := svc.backupSvc.RunForSchedule(jobCtx, preset); err != nil {
3261 return err
3262 }
3263 log.Info().
3264 Str("schedule", string(preset)).
3265 Msg("scheduled backup run complete")
3266 return nil
3267 },
3268 )
3269 if err != nil {
3270 return nil, log.WrapErr(err, "failed to register backup schedule")
3271 }
3272
3273 scheduler.Start(ctx)
3274 log.Info().
3275 Str("schedule", string(preset)).
3276 Msg("backup scheduler enabled")
3277
3278 return scheduler, nil
3279}
3280
3281func resolveBackupSchedule(raw string) (domain.BackupSchedule, error) {
3282 return resolveSchedulePreset(raw, "backups.databases.schedule", domain.ScheduleDaily)
3283}
3284
3285func startVolumeBackupScheduler(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger) (*cronSvc.Scheduler, error) {
3286 if !cfg.Backups.Volumes.Enabled || svc == nil || svc.volumeBackupSvc == nil {
3287 return nil, nil
3288 }
3289
3290 volumeCfg, err := validateVolumeBackupConfig(cfg)
3291 if err != nil {
3292 return nil, err
3293 }
3294
3295 scheduler := cronSvc.NewScheduler(log)
3296 err = scheduler.Add(
3297 "volume-backup-scheduler",
3298 "Volume Backups",
3299 domain.CronSchedule{Interval: volumeCfg.Interval},
3300 func(jobCtx context.Context) error {
3301 if err := svc.volumeBackupSvc.RunVolumeBackupsForSchedule(jobCtx, ""); err != nil {
3302 return err
3303 }
3304 log.Info().
3305 Dur("interval", volumeCfg.Interval).
3306 Msg("scheduled volume backup run complete")
3307 return nil
3308 },
3309 )
3310 if err != nil {
3311 return nil, log.WrapErr(err, "failed to register volume backup schedule")
3312 }
3313
3314 scheduler.Start(ctx)
3315 log.Info().
3316 Dur("interval", volumeCfg.Interval).
3317 Msg("volume backup scheduler enabled")
3318
3319 return scheduler, nil
3320}
3321
3322func startImagePruneScheduler(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger, keepLastGetter func() int) (*cronSvc.Scheduler, error) {
3323 if !cfg.Images.Prune.Enabled || svc == nil || svc.imageSvc == nil {
3324 return nil, nil
3325 }
3326 // The scheduler runs the same use case and planner as manual
3327 // execution. App presence is normal: apps exist on every real
3328 // server, and prune decides per candidate.
3329 if keepLastGetter == nil {
3330 keepLastGetter = func() int { return cfg.Images.Prune.KeepLast }
3331 }
3332 if keepLastGetter() < 0 {
3333 return nil, fmt.Errorf("images.prune.keep_last must be >= 0")
3334 }
3335
3336 preset, err := resolveImagePruneSchedule(cfg.Images.Prune.Schedule)
3337 if err != nil {
3338 return nil, err
3339 }
3340
3341 scheduler := cronSvc.NewScheduler(log)
3342 err = scheduler.Add(
3343 "image-prune",
3344 "Image prune",
3345 domain.CronSchedule{Preset: preset},
3346 func(jobCtx context.Context) error {
3347 keepLast := keepLastGetter()
3348 if keepLast < 0 {
3349 log.Warn().
3350 Int("configured_keep_last", keepLast).
3351 Int("fallback_keep_last", domain.DefaultImagePruneKeepLast).
3352 Msg("invalid images.prune.keep_last; using default")
3353 keepLast = domain.DefaultImagePruneKeepLast
3354 }
3355
3356 report, err := svc.imageSvc.Prune(jobCtx, domain.ImagePruneOptions{
3357 KeepLast: keepLast,
3358 PruneDangling: true,
3359 PruneRegistry: true,
3360 })
3361 if err != nil {
3362 return err
3363 }
3364
3365 log.Info().
3366 Int("keep_last", keepLast).
3367 Int("eligible", report.Plan.CountByVerdict(domain.PruneVerdictEligible)).
3368 Int("protected", report.Plan.CountByVerdict(domain.PruneVerdictProtected)).
3369 Int("unknown", report.Plan.CountByVerdict(domain.PruneVerdictUnknown)).
3370 Int("deleted", len(report.Plan.Deleted)).
3371 Int("failures", len(report.Plan.Failures)).
3372 Int("inventory_gaps", len(report.Plan.Gaps)).
3373 Int("runtime_deleted", report.Runtime.DeletedCount).
3374 Int("registry_tags_removed", report.Registry.TagsRemoved).
3375 Int("registry_blobs_removed", report.Registry.BlobsRemoved).
3376 Msg("scheduled image prune complete")
3377 return nil
3378 },
3379 )
3380 if err != nil {
3381 return nil, log.WrapErr(err, "failed to register image prune schedule")
3382 }
3383
3384 scheduler.Start(ctx)
3385 log.Info().
3386 Str("schedule", string(preset)).
3387 Int("keep_last", keepLastGetter()).
3388 Msg("image prune scheduler enabled")
3389
3390 return scheduler, nil
3391}
3392
3393func resolveImagePruneSchedule(raw string) (domain.BackupSchedule, error) {
3394 return resolveSchedulePreset(raw, "images.prune.schedule", domain.ScheduleDaily)
3395}
3396
3397func resolveSchedulePreset(raw, name string, defaultVal domain.BackupSchedule) (domain.BackupSchedule, error) {
3398 schedule := domain.BackupSchedule(strings.ToLower(strings.TrimSpace(raw)))
3399 if schedule == "" {
3400 schedule = defaultVal
3401 }
3402
3403 switch schedule {
3404 case domain.ScheduleHourly, domain.ScheduleDaily, domain.ScheduleWeekly, domain.ScheduleMonthly:
3405 return schedule, nil
3406 default:
3407 return "", fmt.Errorf("%s must be one of: hourly, daily, weekly, monthly", name)
3408 }
3409}
3410
3411// waitForShutdown blocks on the event loop, handling server errors and
3412// Unix signals (reload, deploy, shutdown) until the context is cancelled.
3413func waitForShutdown(ctx context.Context, errChan <-chan error, reloadChan <-chan os.Signal, reload reloadTrigger, eventBus out.EventBus, log zerowrap.Logger) {
3414 for {
3415 select {
3416 case err := <-errChan:
3417 log.Error().Err(err).Msg("server error")
3418 return
3419 case <-reloadChan:
3420 log.Info().Msg("reload signal received (SIGUSR1)")
3421 _ = reload.Trigger(ctx)
3422 case <-ctx.Done():
3423 log.Info().Msg("shutdown signal received")
3424 return
3425 }
3426 }
3427}
3428
3429// appAdministration is the daemon-owned application lifecycle torn down
3430// during graceful shutdown: it refuses new background deploys, cancels
3431// in-flight executions on the daemon context, and joins them.
3432type appAdministration interface {
3433 Shutdown(context.Context) error
3434}
3435
3436// quiesceAppAdministration cancels and joins daemon-owned app work on the
3437// bounded shutdown context. A non-nil error means the bound expired while an
3438// execution was still unwinding: the caller must stop tearing down the state
3439// and runtime that execution still uses.
3440func quiesceAppAdministration(ctx context.Context, appAdmin appAdministration, log zerowrap.Logger) error {
3441 if appAdmin == nil {
3442 return nil
3443 }
3444 if err := appAdmin.Shutdown(ctx); err != nil {
3445 log.Error().Err(err).Msg("app administration did not quiesce before the shutdown deadline")
3446 return err
3447 }
3448 return nil
3449}
3450
3451// gracefulShutdown stops HTTP servers with a 30s timeout, then shuts down
3452// the container service and cleans up runtime files. It returns an error when
3453// app administration could not quiesce: in that case the remaining teardown
3454// is skipped and the error is propagated to the process entry point, which
3455// exits non-zero instead of closing state or runtime under an in-flight
3456// ExecuteDeploy.
3457func gracefulShutdown(registrySrvs []*http.Server, proxySrv, tlsSrv *http.Server, containerSvc *container.Service, proxySvc *proxy.Service, pkiSvc *pkiusecase.Service, publicTLS in.PublicTLSService, trafficManager *trafficadapter.Manager, monitor *appMonitor, appAdmin appAdministration, appState out.AppState, log zerowrap.Logger) error {
3458 log.Info().Msg("shutting down Gordon...")
3459
3460 // Phase 0: stop periodic app recovery first and wait for any in-flight
3461 // bounded pass, so no reconcile can race runtime, traffic, or state
3462 // teardown (and no goroutine is left behind).
3463 monitor.Stop()
3464
3465 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
3466 defer shutdownCancel()
3467
3468 // Phase 0.5: quiesce daemon-owned app administration. New background
3469 // deploys are refused and in-flight executions are cancelled on the
3470 // daemon context and joined here — before traffic, runtime, or the app
3471 // state store is torn down, so no execution can write to closed state.
3472 // A timeout is fail-closed: the unfinished execution still owns the
3473 // state and runtime it uses, so teardown stops here and the error makes
3474 // the process exit non-zero rather than close resources underneath it.
3475 if err := quiesceAppAdministration(shutdownCtx, appAdmin, log); err != nil {
3476 return fmt.Errorf("app administration quiescence: %w", err)
3477 }
3478
3479 // Phase 1: Stop ingress frontends (TLS, then proxy) — no new traffic accepted
3480 shutdownHTTPServers(shutdownCtx, log, tlsSrv, proxySrv)
3481
3482 if trafficManager != nil {
3483 if err := trafficManager.Shutdown(shutdownCtx); err != nil {
3484 log.Warn().Err(err).Msg("traffic manager shutdown error")
3485 }
3486 }
3487
3488 // Stop PKI maintenance goroutines
3489 if pkiSvc != nil {
3490 pkiSvc.Stop()
3491 }
3492
3493 // Stop public ACME TLS renewal loop
3494 if publicTLS != nil {
3495 if err := publicTLS.Stop(shutdownCtx); err != nil {
3496 log.Warn().Err(err).Msg("public TLS stop error")
3497 }
3498 }
3499
3500 // Phase 2: Drain in-flight registry push sessions before stopping the backend
3501 if proxySvc != nil {
3502 log.Info().Msg("draining in-flight registry requests...")
3503 if drained := proxySvc.DrainRegistryInFlight(25 * time.Second); !drained {
3504 log.Warn().Int64("in_flight", proxySvc.RegistryInFlight()).Msg("registry drain timed out; some in-flight pushes may be interrupted")
3505 }
3506 }
3507
3508 // Phase 3: Stop the registry backends.
3509 shutdownHTTPServers(shutdownCtx, log, registrySrvs...)
3510
3511 if err := containerSvc.Shutdown(shutdownCtx); err != nil {
3512 log.Warn().Err(err).Msg("error during container shutdown")
3513 }
3514 closeAppState(appState, log)
3515
3516 cleanupInternalCredentials()
3517 log.Info().Msg("Gordon stopped")
3518 return nil
3519}
3520
3521// shutdownHTTPServers gracefully stops each non-nil server on the shutdown
3522// context, logging per-server failures so one listener cannot mask another.
3523func shutdownHTTPServers(ctx context.Context, log zerowrap.Logger, servers ...*http.Server) {
3524 for _, srv := range servers {
3525 if srv == nil {
3526 continue
3527 }
3528 if err := srv.Shutdown(ctx); err != nil {
3529 log.Warn().Err(err).Str("addr", srv.Addr).Msg("server shutdown error")
3530 }
3531 }
3532}
3533
3534func closeAppState(state out.AppState, log zerowrap.Logger) {
3535 if state == nil {
3536 return
3537 }
3538 if err := state.Close(); err != nil {
3539 log.Warn().Err(err).Msg("app state close error")
3540 }
3541}
3542
3543// startProxyServers sets up the HTTP proxy server and, when tls_port != 0,
3544// an HTTPS proxy server with on-demand TLS certificates from the internal CA.
3545// certificateSelector implements a multi-source TLS certificate lookup.
3546// Priority: static certs → public ACME TLS → local PKI (internal CA).
3547type certificateSelector struct {
3548 staticCerts []staticTLSCertificate
3549 publicTLS in.PublicTLSService
3550 localPKI *pkiusecase.Service
3551}
3552
3553type staticTLSCertificate struct {
3554 cert tls.Certificate
3555 leaf *x509.Certificate
3556}
3557
3558// GetCertificate selects a TLS certificate based on the ClientHello SNI.
3559//
3560// Priority:
3561// 1. Static certs — exact SNI match (leaf VerifyHostname)
3562// 2. Public ACME TLS — if the host requires ACME coverage
3563// 3. Local PKI (internal CA) — fallback for all other hosts
3564// 4. nil, nil — if no source can serve the host
3565func (s *certificateSelector) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
3566 // 1. Static certs — exact match via leaf VerifyHostname.
3567 if cert := matchingPreparedStaticCert(s.staticCerts, hello.ServerName); cert != nil {
3568 return cert, nil
3569 }
3570
3571 // 2. Public ACME TLS.
3572 if s.publicTLS != nil {
3573 cert, err := s.publicTLS.GetCertificateForHost(hello.ServerName)
3574 if err == nil && cert != nil {
3575 return cert, nil
3576 }
3577 // nil, nil means this host is not an ACME-required route. Errors mean
3578 // public ACME cannot currently serve this host. In both cases, fall
3579 // through to local PKI instead of aborting the TLS handshake.
3580 }
3581
3582 // 3. Local PKI (internal CA).
3583 if s.localPKI != nil {
3584 return s.localPKI.GetCertificate(hello)
3585 }
3586
3587 return nil, nil
3588}
3589
3590func prepareStaticTLSCertificates(certs []tls.Certificate) []staticTLSCertificate {
3591 prepared := make([]staticTLSCertificate, 0, len(certs))
3592 for _, cert := range certs {
3593 if cert.Leaf == nil && len(cert.Certificate) > 0 {
3594 leaf, err := x509.ParseCertificate(cert.Certificate[0])
3595 if err == nil {
3596 cert.Leaf = leaf
3597 }
3598 }
3599 prepared = append(prepared, staticTLSCertificate{cert: cert, leaf: cert.Leaf})
3600 }
3601 return prepared
3602}
3603
3604func matchingPreparedStaticCert(certs []staticTLSCertificate, serverName string) *tls.Certificate {
3605 if serverName == "" {
3606 if len(certs) == 0 {
3607 return nil
3608 }
3609 return &certs[0].cert
3610 }
3611 for i := range certs {
3612 if certs[i].leaf == nil {
3613 continue
3614 }
3615 if err := certs[i].leaf.VerifyHostname(serverName); err == nil {
3616 return &certs[i].cert
3617 }
3618 }
3619 return nil
3620}
3621
3622// matchingStaticCert returns a pointer to the first static certificate whose
3623// leaf verifies the given serverName. Returns nil if no match is found.
3624func matchingStaticCert(certs []tls.Certificate, serverName string) *tls.Certificate {
3625 return matchingPreparedStaticCert(prepareStaticTLSCertificates(certs), serverName)
3626}
3627
3628func startProxyServers(cfg Config, httpHandler, httpsHandler http.Handler, pkiSvc *pkiusecase.Service, publicTLS in.PublicTLSService, trafficManager *trafficadapter.Manager, log zerowrap.Logger) (*http.Server, <-chan struct{}, *http.Server, <-chan struct{}, error) {
3629 var httpSrv *http.Server
3630 var httpReady <-chan struct{}
3631
3632 needsTLS := hasTLSCapableEntrypoint(cfg)
3633 if !hasSmartTCPEntrypoint(cfg) && !needsTLS {
3634 return httpSrv, httpReady, nil, nil, nil
3635 }
3636 cleanupHTTP := func(err error) (*http.Server, <-chan struct{}, *http.Server, <-chan struct{}, error) {
3637 return httpSrv, httpReady, nil, nil, err
3638 }
3639
3640 var tlsConfig *tls.Config
3641 if needsTLS {
3642 var err error
3643 tlsConfig, err = proxyTLSConfig(cfg, pkiSvc, publicTLS, log)
3644 if err != nil {
3645 return cleanupHTTP(err)
3646 }
3647 }
3648 if trafficManager == nil {
3649 return cleanupHTTP(fmt.Errorf("traffic manager is required when traffic entrypoints are enabled"))
3650 }
3651 registerTLSMuxHTTPServers(trafficManager, cfg, httpsHandler, tlsConfig, nil)
3652 registerSmartTCPHTTPServers(trafficManager, cfg, httpHandler, httpsHandler, tlsConfig, nil)
3653
3654 return httpSrv, httpReady, nil, nil, nil
3655}
3656
3657func hasSmartTCPEntrypoint(cfg Config) bool {
3658 for _, entryPoint := range cfg.EntryPoints {
3659 if entryPoint.Protocol == domain.EntryPointProtocolSmartTCP {
3660 return true
3661 }
3662 }
3663 return false
3664}
3665
3666func hasTLSCapableEntrypoint(cfg Config) bool {
3667 for _, entryPoint := range cfg.EntryPoints {
3668 switch entryPoint.Protocol {
3669 case domain.EntryPointProtocolSmartTCP, domain.EntryPointProtocolTLSMux:
3670 return true
3671 }
3672 }
3673 return false
3674}
3675
3676func effectivePublicTLSPort(cfg Config) int {
3677 return effectiveEntrypointPort(cfg, tlsCapableEntryPoint)
3678}
3679
3680func effectiveProxyTLSPort(cfg Config) int {
3681 return effectivePublicTLSPort(cfg)
3682}
3683
3684func effectiveProxyHTTPPort(cfg Config) int {
3685 return effectiveEntrypointPort(cfg, func(protocol domain.EntryPointProtocol) bool {
3686 return protocol == domain.EntryPointProtocolSmartTCP
3687 })
3688}
3689
3690func effectiveEntrypointPort(cfg Config, match func(domain.EntryPointProtocol) bool) int {
3691 if entryPoint, ok := cfg.EntryPoints[traffic.DefaultEdgeEntryPointName]; ok && match(entryPoint.Protocol) {
3692 if port := portFromAddress(entryPoint.Address); port > 0 {
3693 return port
3694 }
3695 }
3696
3697 var candidatePort int
3698 candidates := 0
3699 for name, entryPoint := range cfg.EntryPoints {
3700 if name == traffic.DefaultEdgeEntryPointName || !match(entryPoint.Protocol) {
3701 continue
3702 }
3703 port := portFromAddress(entryPoint.Address)
3704 if port == 0 {
3705 continue
3706 }
3707 candidatePort = port
3708 candidates++
3709 }
3710 if candidates == 1 {
3711 return candidatePort
3712 }
3713 return 0
3714}
3715
3716func tlsCapableEntryPoint(protocol domain.EntryPointProtocol) bool {
3717 switch protocol {
3718 case domain.EntryPointProtocolSmartTCP, domain.EntryPointProtocolTLSMux:
3719 return true
3720 default:
3721 return false
3722 }
3723}
3724
3725func effectiveHTTP01Port(cfg Config) int {
3726 if hasSmartTCPHTTP01Entrypoint(cfg) {
3727 return 80
3728 }
3729 return 0
3730}
3731
3732func validatePublicTLSReadiness(cfg Config) error {
3733 mode, err := domain.ParseACMEChallengeMode(cfg.TLS.ACME.Challenge)
3734 if err != nil {
3735 return err
3736 }
3737 switch mode {
3738 case domain.ACMEChallengeCloudflareDNS01, domain.ACMEChallengeAuto:
3739 return nil
3740 case domain.ACMEChallengeHTTP01:
3741 return validateHTTP01ChallengeReadiness(cfg)
3742 default:
3743 return fmt.Errorf("%w: %q", domain.ErrACMEChallengeInvalid, mode)
3744 }
3745}
3746
3747func validateEffectivePublicTLSReadiness(cfg Config, effective publictls.EffectiveChallenge) error {
3748 if effective.Mode != domain.ACMEChallengeHTTP01 {
3749 return nil
3750 }
3751 return validateHTTP01ChallengeReadiness(cfg)
3752}
3753
3754func validateHTTP01ChallengeReadiness(cfg Config) error {
3755 if hasBoundHTTP01ChallengeListener(cfg) {
3756 return nil
3757 }
3758 return fmt.Errorf("%w: http-01 requires an actually bound HTTP-01 challenge listener on external :80", domain.ErrACMEChallengeInvalid)
3759}
3760
3761func hasBoundHTTP01ChallengeListener(cfg Config) bool {
3762 return hasSmartTCPHTTP01Entrypoint(cfg)
3763}
3764
3765func hasSmartTCPHTTP01Entrypoint(cfg Config) bool {
3766 for _, entryPoint := range cfg.EntryPoints {
3767 if entryPoint.Protocol == domain.EntryPointProtocolSmartTCP && portFromAddress(entryPoint.Address) == 80 {
3768 return true
3769 }
3770 }
3771 return false
3772}
3773
3774func portFromAddress(address string) int {
3775 _, portText, err := net.SplitHostPort(address)
3776 if err != nil {
3777 return 0
3778 }
3779 port, err := strconv.Atoi(portText)
3780 if err != nil {
3781 return 0
3782 }
3783 return port
3784}
3785
3786func proxyTLSConfig(cfg Config, pkiSvc *pkiusecase.Service, publicTLS in.PublicTLSService, log zerowrap.Logger) (*tls.Config, error) {
3787 var staticCerts []tls.Certificate
3788 if cfg.Server.TLSCertFile != "" {
3789 staticCert, err := tls.LoadX509KeyPair(cfg.Server.TLSCertFile, cfg.Server.TLSKeyFile)
3790 if err != nil {
3791 return nil, fmt.Errorf("load TLS keypair: %w", err)
3792 }
3793 staticCerts = []tls.Certificate{staticCert}
3794 log.Info().
3795 Str("cert", cfg.Server.TLSCertFile).
3796 Str("key", cfg.Server.TLSKeyFile).
3797 Msg("loaded static TLS certificate (public ACME and internal CA handle remaining domains)")
3798 }
3799 selector := &certificateSelector{
3800 staticCerts: prepareStaticTLSCertificates(staticCerts),
3801 publicTLS: publicTLS,
3802 localPKI: pkiSvc,
3803 }
3804 return &tls.Config{
3805 MinVersion: tls.VersionTLS12,
3806 GetCertificate: selector.GetCertificate,
3807 NextProtos: []string{"h2", "http/1.1"},
3808 }, nil
3809}
3810
3811func registerSmartTCPHTTPServers(manager *trafficadapter.Manager, cfg Config, httpHandler, httpsHandler http.Handler, tlsConfig *tls.Config, previous map[string]struct{}) map[string]struct{} {
3812 if manager == nil {
3813 return previous
3814 }
3815 next := smartTCPHTTPServerNames(cfg)
3816 for name := range previous {
3817 if _, ok := next[name]; !ok {
3818 manager.SetSmartTCPHTTPServer(name, nil, nil)
3819 manager.SetSmartTCPTLSServer(name, nil, nil)
3820 }
3821 }
3822 var httpProtos http.Protocols
3823 httpProtos.SetHTTP1(true)
3824 httpProtos.SetUnencryptedHTTP2(true)
3825 for name := range next {
3826 if httpHandler != nil {
3827 manager.SetSmartTCPHTTPServer(name, httpHandler, &httpProtos)
3828 }
3829 if httpsHandler != nil && tlsConfig != nil {
3830 manager.SetSmartTCPTLSServer(name, httpsHandler, tlsConfig)
3831 }
3832 }
3833 return next
3834}
3835
3836func smartTCPHTTPServerNames(cfg Config) map[string]struct{} {
3837 names := map[string]struct{}{}
3838 for name, entryPoint := range cfg.EntryPoints {
3839 if entryPoint.Protocol == domain.EntryPointProtocolSmartTCP {
3840 names[name] = struct{}{}
3841 }
3842 }
3843 return names
3844}
3845
3846func registerTLSMuxHTTPServers(manager *trafficadapter.Manager, cfg Config, httpsHandler http.Handler, tlsConfig *tls.Config, previous map[string]struct{}) map[string]struct{} {
3847 if manager == nil {
3848 return previous
3849 }
3850 next := tlsMuxHTTPServerNames(cfg)
3851 for name := range previous {
3852 if _, ok := next[name]; !ok {
3853 manager.SetTLSHTTPServer(name, nil, nil)
3854 }
3855 }
3856 if httpsHandler == nil || tlsConfig == nil {
3857 return next
3858 }
3859 for name := range next {
3860 manager.SetTLSHTTPServer(name, httpsHandler, tlsConfig)
3861 }
3862 return next
3863}
3864
3865func tlsMuxHTTPServerNames(cfg Config) map[string]struct{} {
3866 names := map[string]struct{}{}
3867 for name, entryPoint := range cfg.EntryPoints {
3868 domainEntryPoint := domain.EntryPoint{Name: name, Protocol: entryPoint.Protocol}
3869 if trafficManagerOwnsEntryPoint(domainEntryPoint) && domainEntryPoint.Protocol == domain.EntryPointProtocolTLSMux {
3870 names[name] = struct{}{}
3871 }
3872 }
3873 return names
3874}
3875
3876func startRegistryServers(cfg Config, handler http.Handler, errChan chan<- error, log zerowrap.Logger) (*http.Server, <-chan struct{}, *http.Server, <-chan struct{}) {
3877 port := strconv.Itoa(cfg.Server.RegistryPort)
3878 externalAddr := net.JoinHostPort(cfg.Server.RegistryListenAddr, port)
3879 externalSrv, externalReady := startServer(externalAddr, handler, "registry", nil, errChan, log)
3880 internalReady := closedReadyChannel()
3881 if !needsInternalRegistryListener(cfg.Server.RegistryListenAddr) {
3882 return externalSrv, externalReady, nil, internalReady
3883 }
3884 internalAddr := net.JoinHostPort("127.0.0.1", port)
3885 internalSrv, internalReady := startServer(internalAddr, handler, "internal registry", nil, errChan, log)
3886 return externalSrv, externalReady, internalSrv, internalReady
3887}
3888
3889func closedReadyChannel() <-chan struct{} {
3890 ready := make(chan struct{})
3891 close(ready)
3892 return ready
3893}
3894
3895func needsInternalRegistryListener(listenAddr string) bool {
3896 addr := net.ParseIP(strings.TrimSpace(listenAddr))
3897 if addr == nil {
3898 return listenAddr != ""
3899 }
3900 return !addr.IsLoopback() && !addr.IsUnspecified()
3901}
3902
3903// startServer starts an HTTP server, returning the server instance and a channel
3904// that closes once the listening socket is bound. This lets callers wait for the
3905// port to be ready before taking actions that depend on it (e.g. auto-start
3906// pulling from the local registry). The returned *http.Server can be used for
3907// graceful shutdown.
3908func startServer(addr string, handler http.Handler, name string, protocols *http.Protocols, errChan chan<- error, log zerowrap.Logger) (*http.Server, <-chan struct{}) {
3909 ready := make(chan struct{})
3910
3911 server := &http.Server{
3912 Addr: addr,
3913 Handler: handler,
3914 Protocols: protocols,
3915 ReadHeaderTimeout: 10 * time.Second,
3916 ReadTimeout: 5 * time.Minute,
3917 WriteTimeout: 5 * time.Minute,
3918 IdleTimeout: 120 * time.Second,
3919 MaxHeaderBytes: 1 << 20,
3920 }
3921
3922 go func() {
3923 log.Info().Str("address", addr).Msgf("%s server starting", name)
3924
3925 ln, err := net.Listen("tcp", addr)
3926 if err != nil {
3927 errChan <- fmt.Errorf("%s server error: %w", name, err)
3928 return
3929 }
3930 close(ready) // signal: port is bound and accepting connections
3931
3932 if err := server.Serve(ln); err != nil && err != http.ErrServerClosed {
3933 errChan <- fmt.Errorf("%s server error: %w", name, err)
3934 }
3935 }()
3936
3937 return server, ready
3938}
3939
3940// SendReloadSignal sends SIGUSR1 to the running Gordon process.
3941func SendReloadSignal() error {
3942 process, _, err := findRunningProcess()
3943 if err != nil {
3944 return err
3945 }
3946
3947 if err := process.Signal(syscall.SIGUSR1); err != nil {
3948 return fmt.Errorf("failed to send reload signal: %w", err)
3949 }
3950
3951 return nil
3952}
3953
3954// createPidFile creates a PID file for the Gordon process.
3955// SECURITY: Prefers secure locations (XDG_RUNTIME_DIR, ~/.gordon/run) over /tmp
3956// to prevent symlink attacks and unauthorized access.
3957func createPidFile(log zerowrap.Logger) string {
3958 pid := os.Getpid()
3959
3960 // SECURITY: Prioritize secure locations over /tmp
3961 var locations []string
3962
3963 // Try secure runtime directory first
3964 if runtimeDir, err := getSecureRuntimeDir(); err == nil {
3965 locations = append(locations, filepath.Join(runtimeDir, "gordon.pid"))
3966 }
3967
3968 // Fall back to home directory
3969 if homeDir, err := os.UserHomeDir(); err == nil {
3970 locations = append(locations, filepath.Join(homeDir, ".gordon", "gordon.pid"))
3971 }
3972
3973 // Last resort: /tmp (least secure due to world-writable)
3974 locations = append(locations, filepath.Join(os.TempDir(), "gordon.pid"))
3975
3976 for _, location := range locations {
3977 // Ensure parent directory exists with secure permissions
3978 if err := os.MkdirAll(filepath.Dir(location), 0700); err != nil {
3979 continue
3980 }
3981 if err := os.WriteFile(location, fmt.Appendf(nil, "%d", pid), 0600); err == nil {
3982 log.Debug().Str("pid_file", location).Int("pid", pid).Msg("created PID file")
3983 return location
3984 }
3985 }
3986
3987 log.Warn().Int("pid", pid).Msg("failed to create PID file in any location")
3988 return ""
3989}
3990
3991// removePidFile removes the PID file.
3992func removePidFile(pidFile string, log zerowrap.Logger) {
3993 if err := os.Remove(pidFile); err != nil {
3994 log.Warn().Err(err).Str("pid_file", pidFile).Msg("failed to remove PID file")
3995 } else {
3996 log.Debug().Str("pid_file", pidFile).Msg("removed PID file")
3997 }
3998}
3999
4000func pidFileLocations() []string {
4001 var locations []string
4002 seen := make(map[string]struct{})
4003 add := func(path string) {
4004 if path == "" {
4005 return
4006 }
4007 if _, ok := seen[path]; ok {
4008 return
4009 }
4010 seen[path] = struct{}{}
4011 locations = append(locations, path)
4012 }
4013
4014 // Check secure runtime directory first
4015 if runtimeDir, err := getSecureRuntimeDir(); err == nil {
4016 add(filepath.Join(runtimeDir, "gordon.pid"))
4017 }
4018
4019 // Also check canonical /run/user/<uid> runtime path. This handles cases where
4020 // Gordon started under systemd user services with runtime dir available, but
4021 // CLI invocations (e.g. non-interactive SSH) don't have XDG_RUNTIME_DIR set.
4022 runtimeByUID := filepath.Join("/run/user", strconv.Itoa(os.Getuid()), "gordon", "gordon.pid")
4023 add(runtimeByUID)
4024
4025 // Check explicit XDG_RUNTIME_DIR if present in this process env.
4026 if runtimeDir := os.Getenv("XDG_RUNTIME_DIR"); runtimeDir != "" {
4027 add(filepath.Join(runtimeDir, "gordon.pid"))
4028 }
4029
4030 // Check home directory
4031 if homeDir, err := os.UserHomeDir(); err == nil {
4032 add(filepath.Join(homeDir, ".gordon", "gordon.pid"))
4033 // Legacy location for backward compatibility
4034 add(filepath.Join(homeDir, ".gordon.pid"))
4035 }
4036
4037 // Legacy /tmp locations for backward compatibility
4038 add(filepath.Join(os.TempDir(), "gordon.pid"))
4039 add("/tmp/gordon.pid")
4040
4041 return locations
4042}
4043
4044// findRunningPidFile returns the first PID file whose PID belongs to a live process.
4045// Stale/invalid PID files are ignored and removed when possible.
4046func findRunningPidFile() (string, int, error) {
4047 return findRunningPidFileInLocations(pidFileLocations())
4048}
4049
4050func findRunningPidFileInLocations(locations []string) (string, int, error) {
4051 foundAny := false
4052
4053 for _, location := range locations {
4054 pidBytes, err := os.ReadFile(location)
4055 if err != nil {
4056 continue
4057 }
4058
4059 foundAny = true
4060
4061 var pid int
4062 if _, err := fmt.Sscanf(string(pidBytes), "%d", &pid); err != nil || pid <= 0 {
4063 _ = os.Remove(location)
4064 continue
4065 }
4066
4067 if isProcessAlive(pid) {
4068 return location, pid, nil
4069 }
4070
4071 _ = os.Remove(location)
4072 }
4073
4074 if foundAny {
4075 return "", 0, fmt.Errorf("found stale gordon PID file(s), is Gordon running?")
4076 }
4077
4078 return "", 0, fmt.Errorf("gordon PID file not found, is Gordon running?")
4079}
4080
4081func findRunningProcess() (*os.Process, int, error) {
4082 _, pid, err := findRunningPidFile()
4083 if err != nil {
4084 return nil, 0, err
4085 }
4086
4087 process, err := os.FindProcess(pid)
4088 if err != nil {
4089 return nil, 0, fmt.Errorf("failed to find process: %w", err)
4090 }
4091
4092 return process, pid, nil
4093}
4094
4095func isProcessAlive(pid int) bool {
4096 process, err := os.FindProcess(pid)
4097 if err != nil {
4098 return false
4099 }
4100
4101 err = process.Signal(syscall.Signal(0))
4102 if err == nil {
4103 return true
4104 }
4105
4106 return errors.Is(err, syscall.EPERM)
4107}
4108
4109// loadConfig loads configuration from file and sets defaults.
4110func loadConfig(v *viper.Viper, configPath string) error {
4111 v.SetDefault("server.registry_port", 5000)
4112 v.SetDefault("server.legacy_registry_domains", []string{})
4113 v.SetDefault("server.tls_cert_file", "")
4114 v.SetDefault("server.tls_key_file", "")
4115 v.SetDefault("tls.acme.enabled", false)
4116 v.SetDefault("tls.acme.email", "")
4117 v.SetDefault("tls.acme.challenge", "auto")
4118 v.SetDefault("tls.acme.obtain_batch_size", 1)
4119 v.SetDefault("dns.resolvers", publictls.DefaultDNSResolvers)
4120 v.SetDefault("dns.propagation_timeout", "5m")
4121 v.SetDefault("dns.polling_interval", "5s")
4122 v.SetDefault("server.force_https_redirect", false)
4123 v.SetDefault("server.data_dir", DefaultDataDir())
4124 v.SetDefault("server.runtime", "auto")
4125 v.SetDefault("logging.level", "info")
4126 v.SetDefault("logging.format", "console")
4127 v.SetDefault("logging.file.enabled", false)
4128 v.SetDefault("logging.file.max_size", 100)
4129 v.SetDefault("logging.file.max_backups", 3)
4130 v.SetDefault("logging.file.max_age", 28)
4131 v.SetDefault("logging.container_logs.enabled", true)
4132 v.SetDefault("logging.container_logs.dir", "")
4133 v.SetDefault("logging.container_logs.max_size", 100)
4134 v.SetDefault("logging.container_logs.max_backups", 3)
4135 v.SetDefault("logging.container_logs.max_age", 28)
4136 v.SetDefault("logging.access_log.enabled", false)
4137 v.SetDefault("logging.access_log.format", "json")
4138 v.SetDefault("logging.access_log.output", "stdout")
4139 v.SetDefault("logging.access_log.file_path", "")
4140 v.SetDefault("logging.access_log.max_size", 100)
4141 v.SetDefault("logging.access_log.max_backups", 3)
4142 v.SetDefault("logging.access_log.max_age", 28)
4143 v.SetDefault("logging.access_log.exclude_health_checks", true)
4144 v.SetDefault("logging.access_log.syslog_identifier", "gordon-access")
4145 v.SetDefault("auth.enabled", true)
4146 // Note: auth.type defaults to "token" (the only supported mode)
4147 v.SetDefault("auth.secrets_backend", "")
4148 v.SetDefault("auth.token_expiry", "720h")
4149 v.SetDefault("api.rate_limit.enabled", true)
4150 v.SetDefault("api.rate_limit.global_rps", 500)
4151 v.SetDefault("api.rate_limit.per_ip_rps", 50)
4152 v.SetDefault("api.rate_limit.burst", 100)
4153 v.SetDefault("auto_route.enabled", false)
4154 v.SetDefault("network_isolation.enabled", true)
4155 v.SetDefault("network_isolation.network_prefix", "gordon")
4156 v.SetDefault("network_isolation.internal", false)
4157 v.SetDefault("volumes.auto_create", true)
4158 v.SetDefault("volumes.prefix", "gordon")
4159 v.SetDefault("volumes.preserve", true)
4160 v.SetDefault("backups.databases.enabled", false)
4161 v.SetDefault("backups.databases.schedule", string(domain.ScheduleDaily))
4162 v.SetDefault("backups.databases.storage_dir", "")
4163 v.SetDefault("backups.databases.retention.hourly", 0)
4164 v.SetDefault("backups.databases.retention.daily", 0)
4165 v.SetDefault("backups.databases.retention.weekly", 0)
4166 v.SetDefault("backups.databases.retention.monthly", 0)
4167 v.SetDefault("backups.volumes.enabled", false)
4168 v.SetDefault("backups.volumes.interval", "24h")
4169 v.SetDefault("backups.volumes.compression", string(domain.VolumeBackupCompressionGzip))
4170 v.SetDefault("backups.volumes.timeout", "2h")
4171 v.SetDefault("backups.volumes.max_concurrency", 2)
4172 v.SetDefault("backups.volumes.helper_image", "alpine:3.20")
4173 v.SetDefault("backups.volumes.s3.bucket", "")
4174 v.SetDefault("backups.volumes.s3.region", "")
4175 v.SetDefault("backups.volumes.s3.prefix", "")
4176 v.SetDefault("backups.volumes.s3.endpoint", "")
4177 v.SetDefault("backups.volumes.s3.path_style", false)
4178 v.SetDefault("backups.volumes.s3.sse_algorithm", "")
4179 v.SetDefault("backups.volumes.s3.sse_kms_key_id", "")
4180 v.SetDefault("backups.volumes.retention.keep", 14)
4181 v.SetDefault("images.allowed_registries", []string{})
4182 v.SetDefault("images.require_digest", false)
4183 v.SetDefault("images.prune.enabled", false)
4184 v.SetDefault("images.prune.schedule", string(domain.ScheduleDaily))
4185 v.SetDefault("images.prune.keep_last", domain.DefaultImagePruneKeepLast)
4186 v.SetDefault("containers.security_profile", "compat")
4187 v.SetDefault("telemetry.enabled", false)
4188 v.SetDefault("telemetry.endpoint", "")
4189 v.SetDefault("telemetry.auth_token", "")
4190 v.SetDefault("telemetry.traces", true)
4191 v.SetDefault("telemetry.metrics", true)
4192 v.SetDefault("telemetry.logs", true)
4193 v.SetDefault("telemetry.trace_sample_rate", 1.0)
4194
4195 v.SetDefault("server.max_concurrent_connections", -1) // -1 = use default (10000), 0 = no limit
4196 v.SetDefault("server.registry_allowed_ips", []string{})
4197 v.SetDefault("server.proxy_allowed_ips", []string{})
4198 v.SetDefault("server.registry_listen_address", "")
4199
4200 ConfigureViper(v, configPath)
4201
4202 if err := v.ReadInConfig(); err != nil {
4203 if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
4204 return fmt.Errorf("failed to read config file: %w", err)
4205 }
4206 }
4207
4208 v.SetEnvPrefix("GORDON")
4209 v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
4210 v.AutomaticEnv()
4211
4212 return nil
4213}