1// Package app provides the application initialization and wiring.
2package app
3
4import (
5 "context"
6 "crypto/rand"
7 "crypto/tls"
8 "crypto/x509"
9 "encoding/hex"
10 "encoding/json"
11 "errors"
12 "fmt"
13 "net"
14 "net/http"
15 "os"
16 "os/signal"
17 "path/filepath"
18 "strconv"
19 "strings"
20 "sync"
21 "syscall"
22 "time"
23
24 "github.com/bnema/zerowrap"
25 zerowrapotel "github.com/bnema/zerowrap/otel"
26 "github.com/spf13/viper"
27 "golang.org/x/sys/unix"
28
29 // Adapters - Output
30 "github.com/bnema/gordon/internal/adapters/out/accesslog"
31 acmelego "github.com/bnema/gordon/internal/adapters/out/acmelego"
32 acmestore "github.com/bnema/gordon/internal/adapters/out/acmestore"
33 "github.com/bnema/gordon/internal/adapters/out/docker"
34 "github.com/bnema/gordon/internal/adapters/out/domainsecrets"
35 "github.com/bnema/gordon/internal/adapters/out/envloader"
36 "github.com/bnema/gordon/internal/adapters/out/eventbus"
37 "github.com/bnema/gordon/internal/adapters/out/filesystem"
38 "github.com/bnema/gordon/internal/adapters/out/httpprober"
39 "github.com/bnema/gordon/internal/adapters/out/logwriter"
40 pkiadapter "github.com/bnema/gordon/internal/adapters/out/pki"
41 "github.com/bnema/gordon/internal/adapters/out/ratelimit"
42 s3storage "github.com/bnema/gordon/internal/adapters/out/s3"
43 "github.com/bnema/gordon/internal/adapters/out/secrets"
44 "github.com/bnema/gordon/internal/adapters/out/telemetry"
45 "github.com/bnema/gordon/internal/adapters/out/tokenstore"
46
47 // OTel
48 "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
49
50 // Adapters - Input
51 "github.com/bnema/gordon/internal/adapters/dto"
52 acmehttp "github.com/bnema/gordon/internal/adapters/in/http/acme"
53 "github.com/bnema/gordon/internal/adapters/in/http/admin"
54 authhandler "github.com/bnema/gordon/internal/adapters/in/http/auth"
55 "github.com/bnema/gordon/internal/adapters/in/http/httphelper"
56 "github.com/bnema/gordon/internal/adapters/in/http/middleware"
57 "github.com/bnema/gordon/internal/adapters/in/http/onboarding"
58 proxyadapter "github.com/bnema/gordon/internal/adapters/in/http/proxy"
59 "github.com/bnema/gordon/internal/adapters/in/http/registry"
60 trafficadapter "github.com/bnema/gordon/internal/adapters/in/traffic"
61
62 // Boundaries
63 "github.com/bnema/gordon/internal/boundaries/in"
64 "github.com/bnema/gordon/internal/boundaries/out"
65
66 // Domain
67 "github.com/bnema/gordon/internal/domain"
68
69 // Packages
70 "github.com/bnema/gordon/pkg/version"
71
72 // Use cases
73 "github.com/bnema/gordon/internal/adapters/out/appsecrets"
74 "github.com/bnema/gordon/internal/adapters/out/appstate"
75 "github.com/bnema/gordon/internal/adapters/out/imageref"
76 "github.com/bnema/gordon/internal/usecase/apps"
77 "github.com/bnema/gordon/internal/usecase/apptraffic"
78 "github.com/bnema/gordon/internal/usecase/auth"
79 "github.com/bnema/gordon/internal/usecase/backup"
80 "github.com/bnema/gordon/internal/usecase/config"
81 "github.com/bnema/gordon/internal/usecase/container"
82 cronSvc "github.com/bnema/gordon/internal/usecase/cron"
83 "github.com/bnema/gordon/internal/usecase/deployment"
84 "github.com/bnema/gordon/internal/usecase/health"
85 "github.com/bnema/gordon/internal/usecase/images"
86 "github.com/bnema/gordon/internal/usecase/logs"
87 pkiusecase "github.com/bnema/gordon/internal/usecase/pki"
88 "github.com/bnema/gordon/internal/usecase/proxy"
89 "github.com/bnema/gordon/internal/usecase/publictls"
90 registrySvc "github.com/bnema/gordon/internal/usecase/registry"
91 "github.com/bnema/gordon/internal/usecase/registrystate"
92 secretsSvc "github.com/bnema/gordon/internal/usecase/secrets"
93 servicecfg "github.com/bnema/gordon/internal/usecase/services"
94 "github.com/bnema/gordon/internal/usecase/traffic"
95 volumesSvc "github.com/bnema/gordon/internal/usecase/volumes"
96
97 // Pkg
98 "github.com/bnema/gordon/pkg/bytesize"
99 "github.com/bnema/gordon/pkg/duration"
100)
101
102// Config holds the application configuration.
103type Config struct {
104 Server struct {
105 Port int `mapstructure:"port"`
106 RegistryPort int `mapstructure:"registry_port"`
107 GordonDomain string `mapstructure:"gordon_domain"`
108 RegistryDomain string `mapstructure:"registry_domain"`
109 LegacyRegistryDomains []string `mapstructure:"legacy_registry_domains"`
110 TLSPort int `mapstructure:"tls_port"`
111 TLSCertFile string `mapstructure:"tls_cert_file"`
112 TLSKeyFile string `mapstructure:"tls_key_file"`
113 ForceHTTPSRedirect bool `mapstructure:"force_https_redirect"`
114 DataDir string `mapstructure:"data_dir"`
115 MaxProxyBodySize string `mapstructure:"max_proxy_body_size"` // e.g., "512MB", "1GB"
116 MaxBlobChunkSize string `mapstructure:"max_blob_chunk_size"` // e.g., "512MB", "1GB"
117 MaxBlobSize string `mapstructure:"max_blob_size"` // e.g., "1GB", "2GB"
118 MaxProxyResponseSize string `mapstructure:"max_proxy_response_size"` // e.g., "1GB", "0" for no limit
119 MaxConcurrentConns int `mapstructure:"max_concurrent_connections"`
120 RegistryAllowedIPs []string `mapstructure:"registry_allowed_ips"`
121 ProxyAllowedIPs []string `mapstructure:"proxy_allowed_ips"`
122 RegistryListenAddr string `mapstructure:"registry_listen_address"`
123 } `mapstructure:"server"`
124
125 Apps struct {
126 // RevisionRetention bounds unreferenced-revision GC per app.
127 // Zero or negative restores the compiled default (8).
128 RevisionRetention int `mapstructure:"revision_retention"`
129 } `mapstructure:"apps"`
130
131 // AppMounts declares administrative bind policies keyed by stable mount
132 // name, referenced by name from an app manifest's [[service.bind]].
133 AppMounts map[string]AppMountPolicy `mapstructure:"app_mounts"`
134
135 // AppDevices declares administrative device grants keyed by stable
136 // logical device name, referenced by name from an app manifest's
137 // `devices` list.
138 AppDevices map[string]AppDevicePolicy `mapstructure:"app_devices"`
139
140 Logging struct {
141 Level string `mapstructure:"level"`
142 Format string `mapstructure:"format"`
143 File struct {
144 Enabled bool `mapstructure:"enabled"`
145 Path string `mapstructure:"path"`
146 MaxSize int `mapstructure:"max_size"`
147 MaxBackups int `mapstructure:"max_backups"`
148 MaxAge int `mapstructure:"max_age"`
149 } `mapstructure:"file"`
150 ContainerLogs struct {
151 Enabled bool `mapstructure:"enabled"`
152 Dir string `mapstructure:"dir"`
153 MaxSize int `mapstructure:"max_size"`
154 MaxBackups int `mapstructure:"max_backups"`
155 MaxAge int `mapstructure:"max_age"`
156 } `mapstructure:"container_logs"`
157 AccessLog struct {
158 Enabled bool `mapstructure:"enabled"`
159 Format string `mapstructure:"format"`
160 Output string `mapstructure:"output"`
161 FilePath string `mapstructure:"file_path"`
162 MaxSize int `mapstructure:"max_size"`
163 MaxBackups int `mapstructure:"max_backups"`
164 MaxAge int `mapstructure:"max_age"`
165 ExcludeHealthChecks bool `mapstructure:"exclude_health_checks"`
166 SyslogIdentifier string `mapstructure:"syslog_identifier"`
167 } `mapstructure:"access_log"`
168 } `mapstructure:"logging"`
169
170 Env struct {
171 Dir string `mapstructure:"dir"`
172 } `mapstructure:"env"`
173
174 Volumes struct {
175 AutoCreate bool `mapstructure:"auto_create"`
176 Prefix string `mapstructure:"prefix"`
177 Preserve bool `mapstructure:"preserve"`
178 } `mapstructure:"volumes"`
179
180 Auth struct {
181 Enabled bool `mapstructure:"enabled"`
182 Type string `mapstructure:"type"` // only "token" is supported
183 SecretsBackend string `mapstructure:"secrets_backend"` // "pass", "sops", or "unsafe"
184 Username string `mapstructure:"username"`
185 TokenSecret string `mapstructure:"token_secret"` // path in secrets backend
186 TokenExpiry string `mapstructure:"token_expiry"` // e.g., "720h", "30d"
187 AccessTokenTTL string `mapstructure:"access_token_ttl"` // e.g., "15m", "30m" (default: 15m)
188 } `mapstructure:"auth"`
189
190 API struct {
191 RateLimit struct {
192 Enabled bool `mapstructure:"enabled"`
193 GlobalRPS float64 `mapstructure:"global_rps"`
194 PerIPRPS float64 `mapstructure:"per_ip_rps"`
195 Burst int `mapstructure:"burst"`
196 TrustedProxies []string `mapstructure:"trusted_proxies"`
197 } `mapstructure:"rate_limit"`
198 } `mapstructure:"api"`
199
200 EntryPoints map[string]traffic.EntryPointConfig `mapstructure:"entrypoints"`
201 Traffic traffic.Config `mapstructure:"traffic"`
202 NetworkServices []traffic.NetworkServiceConfig `mapstructure:"network_services"`
203 Services []servicecfg.Config `mapstructure:"services"`
204
205 Backups struct {
206 // Legacy database backup keys. Prefer backups.databases.* for new configs.
207 Enabled bool `mapstructure:"enabled"`
208 Schedule string `mapstructure:"schedule"`
209 StorageDir string `mapstructure:"storage_dir"`
210 Retention struct {
211 Hourly int `mapstructure:"hourly"`
212 Daily int `mapstructure:"daily"`
213 Weekly int `mapstructure:"weekly"`
214 Monthly int `mapstructure:"monthly"`
215 } `mapstructure:"retention"`
216 Databases struct {
217 Enabled bool `mapstructure:"enabled"`
218 Schedule string `mapstructure:"schedule"`
219 StorageDir string `mapstructure:"storage_dir"`
220 Retention struct {
221 Hourly int `mapstructure:"hourly"`
222 Daily int `mapstructure:"daily"`
223 Weekly int `mapstructure:"weekly"`
224 Monthly int `mapstructure:"monthly"`
225 } `mapstructure:"retention"`
226 } `mapstructure:"databases"`
227 Volumes struct {
228 Enabled bool `mapstructure:"enabled"`
229 Interval string `mapstructure:"interval"`
230 Compression string `mapstructure:"compression"`
231 Timeout string `mapstructure:"timeout"`
232 MaxConcurrency int `mapstructure:"max_concurrency"`
233 HelperImage string `mapstructure:"helper_image"`
234 S3 struct {
235 Bucket string `mapstructure:"bucket"`
236 Region string `mapstructure:"region"`
237 Prefix string `mapstructure:"prefix"`
238 Endpoint string `mapstructure:"endpoint"`
239 PathStyle bool `mapstructure:"path_style"`
240 SSEAlgorithm string `mapstructure:"sse_algorithm"`
241 SSEKMSKeyID string `mapstructure:"sse_kms_key_id"`
242 } `mapstructure:"s3"`
243 Retention struct {
244 Keep int `mapstructure:"keep"`
245 } `mapstructure:"retention"`
246 } `mapstructure:"volumes"`
247 } `mapstructure:"backups"`
248
249 Images struct {
250 AllowedRegistries []string `mapstructure:"allowed_registries"`
251 RequireDigest bool `mapstructure:"require_digest"`
252 Prune struct {
253 Enabled bool `mapstructure:"enabled"`
254 Schedule string `mapstructure:"schedule"`
255 KeepLast int `mapstructure:"keep_last"`
256 } `mapstructure:"prune"`
257 } `mapstructure:"images"`
258
259 Containers struct {
260 MemoryLimit string `mapstructure:"memory_limit"` // e.g., "512MB", "1GB"
261 CPULimit float64 `mapstructure:"cpu_limit"` // CPU cores, e.g., 1.0 = 1 core
262 PidsLimit int64 `mapstructure:"pids_limit"` // e.g., 512
263 SecurityProfile string `mapstructure:"security_profile"` // compat or strict
264 } `mapstructure:"containers"`
265
266 NetworkIsolation struct {
267 Enabled bool `mapstructure:"enabled"`
268 Prefix string `mapstructure:"network_prefix"`
269 Internal bool `mapstructure:"internal"`
270 } `mapstructure:"network_isolation"`
271
272 Telemetry telemetry.Config `mapstructure:"telemetry"`
273
274 TLS struct {
275 ACME struct {
276 Enabled bool `mapstructure:"enabled"`
277 Email string `mapstructure:"email"`
278 Challenge string `mapstructure:"challenge"`
279 ObtainBatchSize int `mapstructure:"obtain_batch_size"`
280 } `mapstructure:"acme"`
281 } `mapstructure:"tls"`
282
283 DNS struct {
284 Resolvers []string `mapstructure:"resolvers"`
285 PropagationTimeout string `mapstructure:"propagation_timeout"`
286 PollingInterval string `mapstructure:"polling_interval"`
287 } `mapstructure:"dns"`
288}
289
290// services holds all the services used by the application.
291type services struct {
292 runtime *docker.Runtime
293 eventBus *eventbus.InMemory
294 blobStorage *filesystem.BlobStorage
295 manifestStorage *filesystem.ManifestStorage
296 backupStorage *filesystem.BackupStorage
297 volumeBackupStore out.VolumeBackupStorage
298 volumeBackupCfg domain.VolumeBackupConfig
299 envLoader out.EnvLoader
300 logWriter *logwriter.LogWriter
301 tokenStore out.TokenStore
302 configSvc *config.Service
303 secretSvc *secretsSvc.Service
304 containerSvc *container.Service
305 backupSvc *backup.Service
306 volumeBackupSvc *backup.VolumeService
307 registrySvc *registrySvc.Service
308 healthSvc *health.Service
309 logSvc *logs.Service
310 imageSvc *images.Service
311 volumeSvc *volumesSvc.Service
312 proxySvc *proxy.Service
313 serviceSecretProvider out.SecretProvider
314 authSvc *auth.Service
315 authHandler *authhandler.Handler
316 adminHandler *admin.Handler
317 httpProxyHandler http.Handler
318 httpsProxyHandler http.Handler
319 internalRegUser string
320 internalRegPass string
321 envDir string
322 maxBlobChunkSize int64
323 maxBlobSize int64
324 caAdapter *pkiadapter.CA
325 pkiSvc *pkiusecase.Service
326 appState out.AppState
327 // gcBarrier serializes resource acquisition and its durable
328 // protection publication against destructive prune.
329 gcBarrier out.GCBarrier
330 appDeploySvc *deployment.Service
331 appSvc in.AppService
332 appSvcImpl *apps.AppServiceImpl
333 appActivator *apptraffic.Activator
334 appHostIndex *apptraffic.HostIndex
335 appTrafficPublisher *appTrafficPublisher
336 appMonitor *appMonitor
337 reloadCoordinator *reloadCoordinator
338 publicTLSSvc in.PublicTLSService
339 publicTLSRuntime publicTLSRuntime
340 trafficManager *trafficadapter.Manager
341 tlsHTTPEntryPoints map[string]struct{}
342 smartHTTPEntryPoints map[string]struct{}
343 registryHandler interface {
344 UpdateBlobLimits(maxBlobChunkSize, maxBlobSize int64)
345 }
346}
347
348// Run initializes and starts the Gordon application.
349func Run(ctx context.Context, configPath string) error {
350 // Load configuration
351 v, cfg, err := initConfig(configPath)
352 if err != nil {
353 return err
354 }
355
356 // Initialize logger
357 log, cleanup, err := initLogger(cfg)
358 if err != nil {
359 return err
360 }
361 if cleanup != nil {
362 defer cleanup()
363 }
364
365 ctx = zerowrap.WithCtx(ctx, log)
366
367 // Initialize OpenTelemetry
368 telProvider, telShutdown, err := telemetry.NewProvider(ctx, cfg.Telemetry, "gordon", version.Version())
369 if err != nil {
370 log.Warn().Err(err).Msg("failed to initialize telemetry, continuing without it")
371 } else {
372 // Use a fresh context for shutdown so a canceled app ctx doesn't prevent flushing.
373 defer func() {
374 shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
375 defer cancel()
376 telShutdown(shutdownCtx)
377 }()
378 if cfg.Telemetry.Enabled && cfg.Telemetry.Endpoint != "" {
379 // Bridge zerowrap logs to OTel if log export is enabled
380 if cfg.Telemetry.Logs && telProvider.LogProvider != nil {
381 otelHook := zerowrapotel.NewHookWithProvider(telProvider.LogProvider, "gordon")
382 log = zerowrap.WithHook(log, otelHook)
383 ctx = zerowrap.WithCtx(ctx, log)
384 }
385 log.Info().Str("endpoint", cfg.Telemetry.Endpoint).Msg("telemetry initialized")
386 }
387 }
388
389 log.Info().Msg("Gordon starting")
390
391 warnDeprecatedConfigKeys(v, log)
392
393 // Create PID file
394 pidFile := createPidFile(log)
395 if pidFile != "" {
396 defer removePidFile(pidFile, log)
397 }
398
399 // Create all services
400 svc, err := createServices(ctx, v, cfg, log)
401 if err != nil {
402 return err
403 }
404
405 // Register event handlers
406 cleanupHandlers, err := registerEventHandlers(ctx, svc)
407 if err != nil {
408 return err
409 }
410 // cleanupHandlers is passed into runServers so it can stop debounce
411 // timers before graceful shutdown, preventing deploys during drain.
412
413 // Set up config hot reload
414 if err := setupConfigHotReload(ctx, svc.configSvc, svc.reloadCoordinator); err != nil {
415 return err
416 }
417
418 // Start event bus
419 if err := svc.eventBus.Start(); err != nil {
420 return log.WrapErr(err, "failed to start event bus")
421 }
422 defer svc.eventBus.Stop()
423
424 // Start servers, wait for listeners to bind, then sync/auto-start containers.
425 return runServers(ctx, v, cfg, svc, svc.reloadCoordinator, cleanupHandlers, log)
426}
427
428func warnDeprecatedConfigKeys(v *viper.Viper, log zerowrap.Logger) {
429 for _, key := range []string{"server.tls_enabled", "server.force_hsts"} {
430 if v.IsSet(key) {
431 log.Warn().Str("key", key).Msg("deprecated config key — Gordon now uses an internal CA with automatic TLS; remove this from your config")
432 }
433 }
434}
435
436// initConfig loads configuration from file.
437func initConfig(configPath string) (*viper.Viper, Config, error) {
438 v := viper.New()
439 if err := loadConfig(v, configPath); err != nil {
440 return nil, Config{}, fmt.Errorf("failed to load config: %w", err)
441 }
442
443 var cfg Config
444 if err := v.Unmarshal(&cfg); err != nil {
445 return nil, Config{}, fmt.Errorf("failed to unmarshal config: %w", err)
446 }
447 if err := validateEntrypointMigration(v, cfg); err != nil {
448 return nil, Config{}, err
449 }
450 if err := validateRetiredAppConfig(v); err != nil {
451 return nil, Config{}, err
452 }
453 if _, err := buildAppMountPolicies(cfg); err != nil {
454 return nil, Config{}, err
455 }
456 if err := validateDeviceConfig(v, cfg); err != nil {
457 return nil, Config{}, err
458 }
459
460 return v, cfg, nil
461}
462
463// retiredAppConfigKeys are pre-v3 application keys removed with the
464// declarative-apps cutover. Presence fails boot/reload closed with a
465// config-retired diagnostic naming the fix — never a silent migration.
466var retiredAppConfigKeys = []struct {
467 key string
468 hint string
469}{
470 {"routes", "declare [[service.http]] in an app file, then 'gordon apps apply'"},
471 {"attachments", "declare [[service]] + volumes; attachments are removed"},
472 {"network_groups", "declare [[network.shared]] with ownership verification"},
473 {"services", "one [[service]] per app file"},
474 {"service_routes", "declare [[service.http]] instead"},
475 {"auto", "feature removed; declare explicit interfaces"},
476 {"auto_route", "feature removed; declare explicit interfaces"},
477 {"auto_route_allowed_domains", "feature removed; declare explicit interfaces"},
478 {"previews", "staging is an ordinary app file"},
479}
480
481// validateRetiredAppConfig rejects obsolete application configuration
482// BEFORE any app/runtime mutation, at startup and reload. InConfig
483// (not IsSet) targets explicit file keys only — installation defaults
484// registered via SetDefault must not trip the rejection.
485func validateRetiredAppConfig(v *viper.Viper) error {
486 diagnostics := make([]string, 0, len(retiredAppConfigKeys))
487 for _, retired := range retiredAppConfigKeys {
488 if v.InConfig(retired.key) {
489 diagnostics = append(diagnostics, fmt.Sprintf("key %q was removed in v3; %s", retired.key, retired.hint))
490 }
491 }
492 if len(diagnostics) > 0 {
493 return fmt.Errorf("config-retired: %s", strings.Join(diagnostics, "; "))
494 }
495 return nil
496}
497
498func validateEntrypointMigration(v *viper.Viper, cfg Config) error {
499 if len(cfg.EntryPoints) > 0 {
500 return nil
501 }
502
503 legacyKeys := make([]string, 0, 2)
504 for _, key := range []string{"server.port", "server.tls_port"} {
505 if v.IsSet(key) {
506 legacyKeys = append(legacyKeys, key)
507 }
508 }
509 if len(legacyKeys) == 0 {
510 return nil
511 }
512
513 return fmt.Errorf("legacy %s configuration requires at least one [entrypoints] entry; see docs/upgrading.md", strings.Join(legacyKeys, " or "))
514}
515
516// initLogger initializes the zerowrap logger.
517func initLogger(cfg Config) (zerowrap.Logger, func(), error) {
518 logConfig := zerowrap.Config{
519 Level: cfg.Logging.Level,
520 Format: cfg.Logging.Format,
521 }
522
523 // Always enable file logging so admin API can read process logs
524 // (Config flag is respected for backward-compatibility)
525 logPath := cfg.Logging.File.Path
526 if logPath == "" {
527 dataDir := cfg.Server.DataDir
528 if dataDir == "" {
529 dataDir = DefaultDataDir()
530 }
531 logPath = filepath.Join(dataDir, "logs", "gordon.log")
532 }
533
534 log, cleanup, err := zerowrap.NewWithFile(logConfig, zerowrap.FileConfig{
535 Enabled: cfg.Logging.File.Enabled,
536 Path: logPath,
537 MaxSize: cfg.Logging.File.MaxSize,
538 MaxBackups: cfg.Logging.File.MaxBackups,
539 MaxAge: cfg.Logging.File.MaxAge,
540 Compress: true,
541 })
542 if err != nil {
543 return zerowrap.Default(), nil, fmt.Errorf("failed to create logger with file: %w", err)
544 }
545 return log, cleanup, nil
546}
547
548// initAccessLog creates an access log writer when access logging is enabled.
549// Returns nil, nil when disabled — callers must treat nil writer as "disabled".
550func initAccessLog(cfg Config, log zerowrap.Logger) (*accesslog.Writer, error) {
551 if !cfg.Logging.AccessLog.Enabled {
552 return nil, nil
553 }
554
555 filePath := cfg.Logging.AccessLog.FilePath
556 if filePath == "" && cfg.Logging.AccessLog.Output == "file" {
557 dataDir := cfg.Server.DataDir
558 if dataDir == "" {
559 dataDir = DefaultDataDir()
560 }
561 filePath = filepath.Join(dataDir, "logs", "access.log")
562 }
563
564 writer, err := accesslog.New(accesslog.Config{
565 Format: cfg.Logging.AccessLog.Format,
566 Output: cfg.Logging.AccessLog.Output,
567 FilePath: filePath,
568 MaxSize: cfg.Logging.AccessLog.MaxSize,
569 MaxBackups: cfg.Logging.AccessLog.MaxBackups,
570 MaxAge: cfg.Logging.AccessLog.MaxAge,
571 SyslogIdentifier: cfg.Logging.AccessLog.SyslogIdentifier,
572 })
573 if err != nil {
574 return nil, fmt.Errorf("failed to initialize access log: %w", err)
575 }
576
577 log.Info().
578 Str("format", cfg.Logging.AccessLog.Format).
579 Str("output", cfg.Logging.AccessLog.Output).
580 Msg("access log enabled")
581
582 return writer, nil
583}
584
585// serviceInit holds the shared context for service initialization helpers.
586type serviceInit struct {
587 ctx context.Context
588 v *viper.Viper
589 cfg Config
590 log zerowrap.Logger
591 svc *services
592}
593
594// createServices creates all the application services for server runtime.
595// Public ACME reconciliation is started later, after the HTTP listener is bound.
596func createServices(ctx context.Context, v *viper.Viper, cfg Config, log zerowrap.Logger) (_ *services, retErr error) {
597 return createServicesWithOptions(ctx, v, cfg, log)
598}
599
600// createServicesWithOptions creates all the application services.
601// ACME Reconcile and renewal loop are started later from runServers,
602// after HTTP listeners are bound.
603func createServicesWithOptions(ctx context.Context, v *viper.Viper, cfg Config, log zerowrap.Logger) (_ *services, retErr error) {
604 si := &serviceInit{
605 ctx: ctx,
606 v: v,
607 cfg: cfg,
608 log: log,
609 svc: &services{},
610 }
611 defer func() {
612 if retErr != nil {
613 if si.svc.pkiSvc != nil {
614 si.svc.pkiSvc.Stop()
615 }
616 if si.svc.publicTLSSvc != nil {
617 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
618 defer cancel()
619 if err := si.svc.publicTLSSvc.Stop(ctx); err != nil {
620 si.log.Warn().Err(err).Msg("failed to stop public TLS service during createServices cleanup")
621 }
622 }
623 }
624 }()
625 var err error
626
627 // Create output adapters
628 runtimeSocket := resolveRuntimeConfig(v.GetString("server.runtime"))
629 if si.svc.runtime, si.svc.eventBus, err = createOutputAdapters(ctx, log, runtimeSocket); err != nil {
630 return nil, err
631 }
632
633 // Create storage
634 if si.svc.blobStorage, si.svc.manifestStorage, err = createStorage(cfg, log); err != nil {
635 return nil, err
636 }
637
638 // Create log writer
639 if si.svc.logWriter, err = createLogWriter(cfg, log); err != nil {
640 return nil, err
641 }
642
643 // Create auth service (if enabled)
644 if si.svc.tokenStore, si.svc.authSvc, err = createAuthService(ctx, cfg, log); err != nil {
645 return nil, err
646 }
647
648 if err := setupInternalRegistryAuth(si.svc, log); err != nil {
649 return nil, err
650 }
651
652 // Create config service
653 si.svc.configSvc = config.NewService(v, si.svc.eventBus)
654 if err := si.svc.configSvc.Load(ctx); err != nil {
655 return nil, log.WrapErr(err, "failed to load configuration")
656 }
657
658 if err := si.initPKI(); err != nil {
659 return nil, err
660 }
661
662 if err := si.initSecrets(); err != nil {
663 return nil, err
664 }
665
666 if err := si.initPublicTLS(); err != nil {
667 return nil, err
668 }
669
670 if err := si.initRuntimeProxyAndTraffic(); err != nil {
671 return nil, err
672 }
673
674 si.svc.reloadCoordinator = newReloadCoordinator(v, si.svc.configSvc, si.svc.proxySvc, nil, si.svc.eventBus, si.svc.publicTLSSvc, log)
675 si.registerReloadCoordinatorHooks()
676
677 si.initHandlers()
678
679 return si.svc, nil
680}
681
682// initPKI initialises the internal CA and PKI service when TLS is enabled.
683func (si *serviceInit) initPKI() error {
684 if !hasTLSCapableEntrypoint(si.cfg) {
685 si.log.Info().Msg("internal CA disabled (no TLS-capable entrypoint configured)")
686 return nil
687 }
688
689 if (si.cfg.Server.TLSCertFile == "") != (si.cfg.Server.TLSKeyFile == "") {
690 return fmt.Errorf("both tls_cert_file and tls_key_file must be set, or neither")
691 }
692
693 caAdapter, err := pkiadapter.NewCA(resolveDataDir(si.cfg.Server.DataDir), si.log)
694 if err != nil {
695 return si.log.WrapErr(err, "failed to initialize internal CA")
696 }
697 si.svc.caAdapter = caAdapter
698 si.svc.pkiSvc = pkiusecase.NewService(si.ctx, caAdapter, si.appRoutes(), []string{si.cfg.Server.GordonDomain}, si.log)
699 return nil
700}
701
702// initPublicTLS initializes the public ACME TLS service if enabled.
703func (si *serviceInit) initPublicTLS() error {
704 if !si.cfg.TLS.ACME.Enabled {
705 return nil
706 }
707
708 if err := validatePublicTLSReadiness(si.cfg); err != nil {
709 return err
710 }
711
712 ctx := si.ctx
713 log := si.log
714
715 dnsCfg, err := buildDNSConfig(si.cfg)
716 if err != nil {
717 return log.WrapErr(err, "invalid DNS configuration")
718 }
719
720 publicTLSCfg := publictls.Config{
721 Enabled: si.cfg.TLS.ACME.Enabled,
722 Email: si.cfg.TLS.ACME.Email,
723 Challenge: si.cfg.TLS.ACME.Challenge,
724 HTTPPort: effectiveHTTP01Port(si.cfg),
725 TLSPort: effectivePublicTLSPort(si.cfg),
726 DataDir: resolveDataDir(si.cfg.Server.DataDir),
727 ObtainBatchSize: si.cfg.TLS.ACME.ObtainBatchSize,
728 DNS: dnsCfg,
729 }
730
731 tokenResolver := secrets.NewPublicTLSResolver(secrets.PublicTLSResolverConfig{})
732
733 effective, err := publictls.ResolveEffectiveChallenge(ctx, publicTLSCfg, tokenResolver)
734 if err != nil {
735 return log.WrapErr(err, "resolve ACME challenge")
736 }
737 if err := validateEffectivePublicTLSReadiness(si.cfg, effective); err != nil {
738 return err
739 }
740
741 store, err := acmestore.New(filepath.Join(resolveDataDir(si.cfg.Server.DataDir), "acme"))
742 if err != nil {
743 return log.WrapErr(err, "create ACME store")
744 }
745
746 challenges := publictls.NewHTTP01Challenges()
747
748 var zoneResolver *acmelego.CloudflareZoneResolver
749 if effective.Mode == domain.ACMEChallengeCloudflareDNS01 {
750 zoneResolver = acmelego.NewCloudflareZoneResolver(effective.Token)
751 }
752
753 issuer, err := acmelego.NewIssuer(acmelego.Config{
754 Email: si.cfg.TLS.ACME.Email,
755 Challenge: effective.Mode,
756 Token: effective.Token,
757 Store: store,
758 HTTPChallengeSink: challenges,
759 DNSResolvers: publicTLSCfg.DNS.Resolvers,
760 DNSPropagationTimeout: publicTLSCfg.DNS.PropagationTimeout,
761 DNSPollingInterval: publicTLSCfg.DNS.PollingInterval,
762 })
763 if err != nil {
764 return log.WrapErr(err, "create ACME issuer")
765 }
766
767 svc := publictls.NewService(publicTLSCfg, publictls.ServiceDeps{
768 Routes: si.appRoutes(),
769 Issuer: issuer,
770 Store: store,
771 ZoneResolver: zoneResolver,
772 Challenges: challenges,
773 Effective: effective,
774 AdditionalHosts: []string{si.cfg.Server.GordonDomain},
775 })
776
777 if err := svc.Load(ctx); err != nil {
778 log.Warn().Err(err).Msg("failed to load ACME certificates, continuing")
779 }
780
781 log.Info().
782 Str("email", si.cfg.TLS.ACME.Email).
783 Str("challenge", string(effective.Mode)).
784 Msg("public ACME TLS initialized (runtime start deferred)")
785
786 si.svc.publicTLSSvc = svc
787 si.svc.publicTLSRuntime = svc
788 return nil
789}
790
791// publicTLSRuntime is the subset of in.PublicTLSService needed at server runtime
792// after the HTTP listener is bound: reconcile missing certs and start the renewal
793// loop. It is nil when ACME is disabled.
794type publicTLSRuntime interface {
795 Reconcile(context.Context) error
796 StartRenewalLoop(context.Context, time.Duration) <-chan struct{}
797}
798
799func startPublicTLSRuntime(ctx context.Context, svc publicTLSRuntime, log zerowrap.Logger) error {
800 if svc == nil {
801 return nil
802 }
803 reconcileErr := svc.Reconcile(ctx)
804 svc.StartRenewalLoop(ctx, time.Hour)
805 log.Info().Msg("public ACME TLS runtime started")
806 return reconcileErr
807}
808
809// initSecrets creates the domain secret store, env loader, and secret service.
810func (si *serviceInit) initSecrets() error {
811 envDir, backend, passStore, domainSecretStore, err := createDomainSecretStore(si.cfg, si.log)
812 if err != nil {
813 return err
814 }
815 si.svc.envDir = envDir
816
817 if si.svc.envLoader, err = createEnvLoader(backend, envDir, passStore, si.log); err != nil {
818 return err
819 }
820
821 si.svc.serviceSecretProvider = createStandaloneServiceSecretProvider(backend, resolveDataDir(si.cfg.Server.DataDir), si.log)
822 si.svc.secretSvc = secretsSvc.NewService(domainSecretStore, si.log, si.svc.eventBus)
823 return nil
824}
825
826// initRuntimeAndProxy creates container, backup, registry, image, volume, and proxy services.
827func (si *serviceInit) initRuntimeProxyAndTraffic() error {
828 if err := si.initRuntimeAndProxy(); err != nil {
829 return err
830 }
831 si.svc.trafficManager = trafficadapter.NewManager()
832 return si.initApps()
833}
834
835func containerResourceLimits(cfg Config) (deployment.ResourceLimits, error) {
836 limits := deployment.ResourceLimits{PidsLimit: cfg.Containers.PidsLimit}
837 if cfg.Containers.MemoryLimit != "" {
838 parsed, err := bytesize.Parse(cfg.Containers.MemoryLimit)
839 if err != nil {
840 return limits, fmt.Errorf("invalid containers.memory_limit %q: %w", cfg.Containers.MemoryLimit, err)
841 }
842 limits.MemoryBytes = parsed
843 }
844 if cfg.Containers.CPULimit > 0 {
845 limits.NanoCPUs = int64(cfg.Containers.CPULimit * float64(time.Second))
846 }
847 return limits, nil
848}
849
850// appDeployEngine is the deployment-engine subset the daemon-owned app
851// service executes through. It mirrors the usecase's own port so the
852// composition root stays decoupled from engine internals.
853type appDeployEngine interface {
854 StartDeploy(ctx context.Context, input deployment.DeployInput) (*deployment.StartDeployResult, error)
855 AbandonDeploy(ctx context.Context, claim deployment.DeployClaim) error
856 ExecuteDeploy(ctx context.Context, claim deployment.DeployClaim) (*deployment.DeployResult, error)
857 Stop(ctx context.Context, app, opID string) (*deployment.LifecycleResult, error)
858 Start(ctx context.Context, app, opID string) (*deployment.LifecycleResult, error)
859 Restart(ctx context.Context, app, service, opID string) (*deployment.LifecycleResult, error)
860 Remove(ctx context.Context, app, opID string) (*deployment.LifecycleResult, error)
861}
862
863// newAppDaemonService builds the daemon-owned app administration service.
864// ctx is the daemon/supervisor lifecycle context: background deploy
865// executions derive from it and are cancelled by Shutdown during graceful
866// teardown. It is deliberately never an HTTP request context, so request
867// cancellation cannot abort an in-flight replacement.
868func newAppDaemonService(ctx context.Context, store out.AppState, deploy appDeployEngine, secrets out.SecretWriter, log zerowrap.Logger) *apps.AppServiceImpl {
869 return apps.NewAppServiceImpl(store, deploy, secrets, log).WithDaemonContext(ctx)
870}
871
872// initApps wires the single v3 app engine: bbolt state, image digests,
873// pass-backed secrets, deployment/lifecycle execution, and traffic
874// activation. The daemon is the sole app-state writer; CLI reaches the
875// engine only through the admin /apps surface.
876func (si *serviceInit) initApps() error {
877 dataDir := resolveDataDir(si.cfg.Server.DataDir)
878 store, err := appstate.NewStore(dataDir, si.log)
879 if err != nil {
880 return si.log.WrapErr(err, "failed to open app state")
881 }
882 store.WithRevisionRetention(si.cfg.Apps.RevisionRetention)
883 si.svc.appState = store
884 // One process-wide GC barrier owns the ordering between resource
885 // acquisition/publication and destructive prune.
886 si.svc.gcBarrier = newGCBarrier()
887 if si.svc.backupSvc != nil {
888 si.svc.backupSvc.WithAppState(store)
889 }
890 if si.svc.volumeBackupSvc != nil {
891 si.svc.volumeBackupSvc.WithAppState(store)
892 }
893 if si.svc.imageSvc != nil {
894 si.svc.imageSvc.WithPrunePorts(store, si.svc.runtime, si.svc.gcBarrier)
895 }
896 if si.svc.volumeSvc != nil {
897 si.svc.volumeSvc.WithPrunePorts(store, si.svc.runtime, si.svc.gcBarrier)
898 }
899 registryDomain := si.svc.configSvc.GetRegistryDomain()
900 imagePolicy := domain.ImageSourcePolicy{
901 AllowedRegistries: si.cfg.Images.AllowedRegistries,
902 RequireDigest: si.cfg.Images.RequireDigest,
903 InstallationRegistry: registryDomain,
904 }
905 if err := imagePolicy.Validate(); err != nil {
906 return fmt.Errorf("invalid image registry policy: %w", err)
907 }
908 resolver := imageref.NewResolver(registryDomain, si.svc.manifestStorage, nil).WithPolicy(imagePolicy)
909 // The publisher is the single serialized HTTP/L4 boundary: raw
910 // activations, fail-closed recovery withdrawal, and reload all go
911 // through it. It resolves writers/permission paths late-bound.
912 si.svc.appTrafficPublisher = newAppTrafficPublisher(si.svc, si.cfg)
913 limits, err := containerResourceLimits(si.cfg)
914 if err != nil {
915 return err
916 }
917 si.svc.appDeploySvc = deployment.NewService(deployment.Deps{
918 State: store,
919 Runtime: si.svc.runtime,
920 Images: resolver,
921 Secrets: si.svc.serviceSecretProvider,
922 Registry: deployment.RegistryConfig{
923 Domain: registryDomain,
924 PullAddress: net.JoinHostPort("127.0.0.1", strconv.Itoa(si.svc.configSvc.GetRegistryPort())),
925 Username: si.svc.internalRegUser,
926 Password: si.svc.internalRegPass,
927 },
928 Networks: deployment.NetworkConfig{
929 Prefix: si.cfg.NetworkIsolation.Prefix,
930 Internal: si.cfg.NetworkIsolation.Internal,
931 },
932 Limits: limits,
933 ImagePolicy: imagePolicy,
934 Traffic: si.svc.appTrafficPublisher,
935 }, si.log).WithGCBarrier(si.svc.gcBarrier)
936 si.svc.appActivator = apptraffic.NewActivator(si.log)
937 mountPolicies, err := buildAppMountPolicies(si.cfg)
938 if err != nil {
939 return err
940 }
941 devicePolicies, err := buildAppDevicePolicies(si.cfg)
942 if err != nil {
943 return err
944 }
945 si.svc.appDeploySvc.SetBindPolicies(mountPolicies)
946 si.svc.appDeploySvc.SetDevicePolicies(devicePolicies)
947 appSvcImpl := newAppDaemonService(si.ctx, store, si.svc.appDeploySvc, appsecrets.NewStore(si.log), si.log).
948 WithEntrypoints(appEntrypointListeners(si.cfg)).
949 WithGCBarrier(si.svc.gcBarrier).
950 WithImagePolicy(imagePolicy).
951 WithBindPolicies(mountPolicies).
952 WithDevicePolicies(devicePolicies)
953 si.svc.appSvcImpl = appSvcImpl
954 si.svc.appSvc = appSvcImpl
955 // Health checks resolve from ACTIVE state (loopback backends).
956 si.svc.healthSvc = health.NewService(store, si.svc.runtime, httpprober.New(), si.log)
957 // ACTIVE-derived host index for the proxy: rebuilt after every
958 // activation and at boot. Wired here (store is open); the proxy
959 // itself was built earlier in initRuntimeAndProxy.
960 si.svc.appHostIndex = apptraffic.NewHostIndex()
961 if si.svc.proxySvc != nil {
962 si.svc.proxySvc.WithAppTargets(si.svc.appHostIndex)
963 }
964 return nil
965}
966
967// appRouteSource adapts the ACTIVE-derived host index plus installation
968// external routes to the route-source interfaces (PKI AppRoutes,
969// public-TLS RouteSource). bbolt stays authoritative; this adapter only
970// projects. Fail-closed: unwired index yields zero app hosts. The index
971// is late-bound (built after PKI/public-TLS), so it resolves per call.
972type appRouteSource struct {
973 index func() *apptraffic.HostIndex
974 external func() map[string]string
975}
976
977// AppHosts implements out.AppHostSource.
978func (s *appRouteSource) AppHosts() []out.AppHost {
979 if s == nil || s.index == nil {
980 return nil
981 }
982 index := s.index()
983 if index == nil {
984 return nil
985 }
986 return index.AppHosts()
987}
988
989// GetExternalRoutes implements the installation half of out.AppRoutes
990// and the public-TLS RouteSource.
991func (s *appRouteSource) GetExternalRoutes() map[string]string {
992 if s == nil || s.external == nil {
993 return nil
994 }
995 return s.external()
996}
997
998// GetRoutes implements the public-TLS RouteSource as a consumer-local
999// adapter: host-only entries, Image explicitly unused (no app meaning).
1000func (s *appRouteSource) GetRoutes(context.Context) []domain.Route {
1001 hosts := s.AppHosts()
1002 routes := make([]domain.Route, 0, len(hosts))
1003 for _, h := range hosts {
1004 routes = append(routes, domain.Route{Domain: h.Host})
1005 }
1006 return routes
1007}
1008
1009// appRoutes returns the shared ACTIVE-derived route source, late-bound
1010// to the host index built in initApps.
1011func (si *serviceInit) appRoutes() *appRouteSource {
1012 return &appRouteSource{
1013 index: func() *apptraffic.HostIndex { return si.svc.appHostIndex },
1014 external: func() map[string]string { return si.svc.configSvc.GetExternalRoutes() },
1015 }
1016}
1017
1018// appEntrypointPolicies maps installation entrypoints to the projection
1019// policy (trusted CIDRs, raw-fallback behavior, transport limits).
1020func appEntrypointPolicies(cfg Config) map[string]apptraffic.EntrypointPolicy {
1021 policies := make(map[string]apptraffic.EntrypointPolicy, len(cfg.EntryPoints))
1022 for name, entry := range cfg.EntryPoints {
1023 policies[name] = apptraffic.EntrypointPolicy{
1024 Name: name,
1025 Address: entry.Address,
1026 Protocol: entry.Protocol,
1027 TrustedCIDRs: entry.TrustedCIDRs,
1028 RawFallback: entry.RawFallback,
1029 RawFallbackTrustedCIDRs: entry.RawFallbackTrustedCIDRs,
1030 AllowPublicRawFallback: entry.AllowPublicRawFallback,
1031 }
1032 }
1033 return policies
1034}
1035
1036// appEntrypointListeners maps installation entrypoints to the canonical
1037// listener an app L4 publish declaration must match exactly.
1038func appEntrypointListeners(cfg Config) map[string]domain.EntryPointListener {
1039 listeners := make(map[string]domain.EntryPointListener, len(cfg.EntryPoints))
1040 for name, entry := range cfg.EntryPoints {
1041 listeners[name] = domain.EntryPointListener{Address: entry.Address, Protocol: entry.Protocol}
1042 }
1043 return listeners
1044}
1045
1046// rebuildAppHostIndex re-projects ACTIVE state into the proxy host index
1047// and applies the full HTTP/L4 traffic graph through the serialized
1048// publisher. Call after every activation (deploy/start/restart/stop/
1049// remove) and at boot; callers decide whether a failure is fatal to the
1050// mutation.
1051func rebuildAppHostIndex(ctx context.Context, svc *services) error {
1052 if svc.appTrafficPublisher == nil {
1053 return nil
1054 }
1055 return svc.appTrafficPublisher.RebuildTraffic(ctx)
1056}
1057
1058func (si *serviceInit) initRuntimeAndProxy() error {
1059 var err error
1060
1061 if si.svc.containerSvc, err = createContainerService(si.ctx, si.v, si.cfg, si.svc, si.log); err != nil {
1062 return err
1063 }
1064
1065 if si.svc.backupStorage, si.svc.backupSvc, err = createBackupService(si.cfg, si.svc, si.log); err != nil {
1066 return err
1067 }
1068 if si.svc.volumeBackupStore, si.svc.volumeBackupSvc, si.svc.volumeBackupCfg, err = createVolumeBackupService(si.ctx, si.cfg, si.svc, si.log); err != nil {
1069 return err
1070 }
1071
1072 registryState := registrystate.New()
1073 si.svc.registrySvc = registrySvc.NewService(si.svc.blobStorage, si.svc.manifestStorage, si.svc.eventBus, registryState)
1074 si.svc.imageSvc = images.NewService(si.svc.runtime, si.svc.manifestStorage, si.svc.blobStorage, si.log, registryState)
1075 si.svc.volumeSvc = volumesSvc.NewService(si.svc.runtime)
1076
1077 injectTelemetryMetrics(si.cfg, si.svc, si.log)
1078
1079 proxyCfg, err := buildProxyConfig(si.cfg, si.log)
1080 if err != nil {
1081 return err
1082 }
1083 si.svc.maxBlobChunkSize = proxyCfg.maxBlobChunkSize
1084 si.svc.maxBlobSize = proxyCfg.maxBlobSize
1085 si.svc.proxySvc = proxy.NewService(si.svc.configSvc, proxyCfg.proxyConfig)
1086
1087 // Wire synchronous proxy cache invalidation for zero-downtime deployments.
1088 return nil
1089}
1090
1091// initHandlers creates the auth, health, log, preview, and admin handlers.
1092func (si *serviceInit) registerReloadCoordinatorHooks() {
1093 if si.svc.reloadCoordinator == nil {
1094 return
1095 }
1096 if si.svc.appSvcImpl != nil {
1097 si.svc.reloadCoordinator.SetAppMountPoliciesApplier(func(_ context.Context, reloadCfg Config) error {
1098 policies, err := buildAppMountPolicies(reloadCfg)
1099 if err != nil {
1100 return err
1101 }
1102 devicePolicies, err := buildAppDevicePolicies(reloadCfg)
1103 if err != nil {
1104 return err
1105 }
1106 si.svc.appSvcImpl.SetBindPolicies(policies)
1107 si.svc.appSvcImpl.SetDevicePolicies(devicePolicies)
1108 if si.svc.appDeploySvc != nil {
1109 si.svc.appDeploySvc.SetBindPolicies(policies)
1110 si.svc.appDeploySvc.SetDevicePolicies(devicePolicies)
1111 }
1112 return nil
1113 })
1114 }
1115 if si.svc.containerSvc == nil {
1116 return
1117 }
1118
1119 si.svc.reloadCoordinator.SetContainerConfigApplier(func(reloadCtx context.Context, reloadCfg Config) error {
1120 containerCfg, err := buildContainerServiceConfig(reloadCtx, si.v, reloadCfg, si.svc, si.log)
1121 if err != nil {
1122 return err
1123 }
1124 managementHosts := []string{reloadCfg.Server.GordonDomain}
1125 if si.svc.pkiSvc != nil {
1126 si.svc.pkiSvc.SetAdditionalDomains(managementHosts)
1127 }
1128 if si.svc.publicTLSSvc != nil {
1129 si.svc.publicTLSSvc.SetAdditionalHosts(reloadCtx, managementHosts)
1130 }
1131 var tlsConfig *tls.Config
1132 if hasTLSCapableEntrypoint(reloadCfg) && si.svc.httpsProxyHandler != nil {
1133 var tlsErr error
1134 tlsConfig, tlsErr = proxyTLSConfig(reloadCfg, si.svc.pkiSvc, si.svc.publicTLSSvc, si.log)
1135 if tlsErr != nil {
1136 return tlsErr
1137 }
1138 }
1139 if err := si.svc.appTrafficPublisher.RebuildWithConfig(reloadCtx, reloadCfg); err != nil {
1140 return err
1141 }
1142 si.svc.tlsHTTPEntryPoints = registerTLSMuxHTTPServers(si.svc.trafficManager, reloadCfg, si.svc.httpsProxyHandler, tlsConfig, si.svc.tlsHTTPEntryPoints)
1143 si.svc.smartHTTPEntryPoints = registerSmartTCPHTTPServers(si.svc.trafficManager, reloadCfg, si.svc.httpProxyHandler, si.svc.httpsProxyHandler, tlsConfig, si.svc.smartHTTPEntryPoints)
1144 si.svc.containerSvc.UpdateConfig(containerCfg)
1145 return nil
1146 })
1147}
1148
1149func (si *serviceInit) initHandlers() {
1150 if si.svc.authSvc != nil {
1151 internalAuth := authhandler.InternalAuth{
1152 Username: si.svc.internalRegUser,
1153 Password: si.svc.internalRegPass,
1154 }
1155 si.svc.authHandler = authhandler.NewHandler(si.svc.authSvc, internalAuth, si.log)
1156 }
1157
1158 si.svc.logSvc = logs.NewService(resolveLogFilePath(si.cfg), si.cfg.Logging.File.Enabled, si.svc.runtime, si.log)
1159 // initApps (via initRuntimeProxyAndTraffic) runs before initHandlers,
1160 // so durable ACTIVE state is already open here.
1161 if si.svc.appState != nil {
1162 si.svc.logSvc.WithAppState(si.svc.appState)
1163 }
1164
1165 if si.svc.trafficManager == nil {
1166 si.svc.trafficManager = trafficadapter.NewManager()
1167 }
1168
1169 si.svc.adminHandler = admin.NewHandler(admin.HandlerDeps{
1170 ConfigSvc: si.svc.configSvc,
1171 AuthSvc: si.svc.authSvc,
1172 ContainerSvc: si.svc.containerSvc,
1173 HealthSvc: si.svc.healthSvc,
1174 SecretSvc: si.svc.secretSvc,
1175 LogSvc: si.svc.logSvc,
1176 RegistrySvc: si.svc.registrySvc,
1177 ReloadTrigger: si.svc.reloadCoordinator,
1178 Log: si.log,
1179 BackupSvc: si.svc.backupSvc,
1180 VolumeBackupSvc: si.svc.volumeBackupSvc,
1181 ImageSvc: si.svc.imageSvc,
1182 VolumeSvc: si.svc.volumeSvc,
1183 PublicTLSSvc: si.svc.publicTLSSvc,
1184 TrafficSvc: si.svc.trafficManager,
1185 AppSvc: si.svc.appSvc,
1186 })
1187}
1188
1189// injectTelemetryMetrics creates and injects OTel metrics into services when
1190// telemetry is enabled. Skipped otherwise to avoid unnecessary allocations.
1191func injectTelemetryMetrics(cfg Config, svc *services, log zerowrap.Logger) {
1192 if !cfg.Telemetry.Enabled || !cfg.Telemetry.Metrics {
1193 return
1194 }
1195 gordonMetrics, err := telemetry.NewMetrics()
1196 if err != nil {
1197 log.Warn().Err(err).Msg("failed to create telemetry metrics, continuing without metrics")
1198 return
1199 }
1200 svc.registrySvc.SetMetrics(gordonMetrics)
1201 svc.eventBus.SetMetrics(gordonMetrics)
1202}
1203
1204func setupInternalRegistryAuth(svc *services, log zerowrap.Logger) error {
1205 var err error
1206 svc.internalRegUser, svc.internalRegPass, err = generateInternalRegistryAuth()
1207 if err != nil {
1208 return log.WrapErr(err, "failed to generate internal registry credentials")
1209 }
1210
1211 // Persist credentials to file for CLI access (gordon auth internal)
1212 if err := persistInternalCredentials(svc.internalRegUser, svc.internalRegPass); err != nil {
1213 log.Warn().Err(err).Msg("failed to persist internal credentials for CLI access")
1214 }
1215
1216 log.Debug().Msg("internal registry auth generated for loopback pulls")
1217 return nil
1218}
1219
1220func createDomainSecretStore(cfg Config, log zerowrap.Logger) (string, domain.SecretsBackend, *domainsecrets.PassStore, out.DomainSecretStore, error) {
1221 envDir := resolveEnvDir(cfg)
1222 backend, err := resolveSecretsBackend(cfg.Auth.SecretsBackend)
1223 if err != nil {
1224 return "", "", nil, nil, log.WrapErr(err, "failed to resolve secrets backend")
1225 }
1226
1227 switch backend {
1228 case domain.SecretsBackendPass:
1229 passStore, err := domainsecrets.NewPassStore(log)
1230 if err != nil {
1231 return "", backend, nil, nil, log.WrapErr(err, "failed to create pass domain secret store")
1232 }
1233 if err := migrateEnvFilesToPass(envDir, passStore, log); err != nil {
1234 return "", backend, nil, nil, log.WrapErr(err, "failed to migrate env files to pass")
1235 }
1236 return envDir, backend, passStore, passStore, nil
1237 default:
1238 store, err := domainsecrets.NewFileStore(envDir, log)
1239 if err != nil {
1240 return "", backend, nil, nil, log.WrapErr(err, "failed to create domain secret store")
1241 }
1242 return envDir, backend, nil, store, nil
1243 }
1244}
1245
1246// resolveLogFilePath returns the configured log file path or a default.
1247func resolveLogFilePath(cfg Config) string {
1248 if cfg.Logging.File.Path != "" {
1249 return cfg.Logging.File.Path
1250 }
1251 dataDir := cfg.Server.DataDir
1252 if dataDir == "" {
1253 dataDir = DefaultDataDir()
1254 }
1255 return filepath.Join(dataDir, "logs", "gordon.log")
1256}
1257
1258// resolveRuntimeConfig converts a server.runtime config value to a socket path.
1259// "auto" or "" means auto-detect.
1260// Named runtimes ("podman", "docker") are resolved to well-known socket paths.
1261// URI schemes (unix://) are stripped so callers receive a bare path.
1262func resolveRuntimeConfig(value string) string {
1263 if value == "" || value == "auto" {
1264 return ""
1265 }
1266 // Named runtimes: resolve to well-known socket paths.
1267 switch value {
1268 case "podman":
1269 // Check XDG_RUNTIME_DIR first (rootless Podman).
1270 if xdg := os.Getenv("XDG_RUNTIME_DIR"); xdg != "" {
1271 candidate := filepath.Join(xdg, "podman", "podman.sock")
1272 if _, err := os.Stat(candidate); err == nil {
1273 return candidate
1274 }
1275 }
1276 // Fallback to system-wide Podman socket.
1277 return "/run/podman/podman.sock"
1278 case "docker":
1279 return "/var/run/docker.sock"
1280 }
1281 // Explicit socket path — strip URI scheme if present.
1282 if socketPath, ok := strings.CutPrefix(value, "unix://"); ok {
1283 return socketPath
1284 }
1285 return value
1286}
1287
1288// createOutputAdapters creates the container runtime and event bus.
1289func createOutputAdapters(ctx context.Context, log zerowrap.Logger, runtimeSocket string) (*docker.Runtime, *eventbus.InMemory, error) {
1290 detection := docker.DetectRuntimeSocket(runtimeSocket)
1291
1292 var runtime *docker.Runtime
1293 var err error
1294
1295 switch detection.Source {
1296 case "none":
1297 return nil, nil, fmt.Errorf("no container runtime found: checked Docker socket, Podman socket, DOCKER_HOST env var. Install Docker or Podman, or set server.runtime in config")
1298 case "DOCKER_HOST_passthrough":
1299 detection.RuntimeName = "docker"
1300 runtime, err = docker.NewRuntime()
1301 default:
1302 if detection.SocketPath != "" {
1303 runtime, err = docker.NewRuntimeWithSocket(detection.SocketPath)
1304 } else {
1305 runtime, err = docker.NewRuntime()
1306 }
1307 }
1308 if err != nil {
1309 return nil, nil, log.WrapErr(err, "failed to create container runtime")
1310 }
1311
1312 if err := runtime.Ping(ctx); err != nil {
1313 return nil, nil, log.WrapErr(err, fmt.Sprintf("container runtime not available (detected: %s via %s)", detection.RuntimeName, detection.Source))
1314 }
1315
1316 runtimeVersion, _ := runtime.Version(ctx)
1317 log.Info().
1318 Str("runtime", detection.RuntimeName).
1319 Str("version", runtimeVersion).
1320 Str("source", detection.Source).
1321 Msg("container runtime initialized")
1322
1323 eventBus := eventbus.NewInMemory(100, log)
1324
1325 return runtime, eventBus, nil
1326}
1327
1328// createStorage creates blob and manifest storage.
1329func createStorage(cfg Config, log zerowrap.Logger) (*filesystem.BlobStorage, *filesystem.ManifestStorage, error) {
1330 dataDir := cfg.Server.DataDir
1331 if dataDir == "" {
1332 dataDir = DefaultDataDir()
1333 }
1334
1335 registryDir := filepath.Join(dataDir, "registry")
1336
1337 blobStorage, err := filesystem.NewBlobStorage(registryDir, log)
1338 if err != nil {
1339 return nil, nil, log.WrapErr(err, "failed to create blob storage")
1340 }
1341
1342 manifestStorage, err := filesystem.NewManifestStorage(registryDir, log)
1343 if err != nil {
1344 return nil, nil, log.WrapErr(err, "failed to create manifest storage")
1345 }
1346
1347 return blobStorage, manifestStorage, nil
1348}
1349
1350// createEnvLoader creates the environment loader with secret providers.
1351func createEnvLoader(backend domain.SecretsBackend, envDir string, passStore *domainsecrets.PassStore, log zerowrap.Logger) (out.EnvLoader, error) {
1352 switch backend {
1353 case domain.SecretsBackendPass:
1354 loader, err := envloader.NewPassLoader(passStore, log)
1355 if err != nil {
1356 return nil, log.WrapErr(err, "failed to create pass env loader")
1357 }
1358 return loader, nil
1359 default:
1360 loader, err := envloader.NewFileLoader(envDir, log)
1361 if err != nil {
1362 return nil, log.WrapErr(err, "failed to create env loader")
1363 }
1364
1365 // Register secret providers
1366 passProvider := secrets.NewPassProvider(log)
1367 if passProvider.IsAvailable() {
1368 loader.RegisterSecretProvider(passProvider)
1369 log.Debug().Msg("pass secret provider registered")
1370 }
1371
1372 sopsProvider := secrets.NewSopsProvider(log)
1373 if sopsProvider.IsAvailable() {
1374 loader.RegisterSecretProvider(sopsProvider)
1375 log.Debug().Msg("sops secret provider registered")
1376 }
1377
1378 return loader, nil
1379 }
1380}
1381
1382// createLogWriter creates the container log writer.
1383func createLogWriter(cfg Config, log zerowrap.Logger) (*logwriter.LogWriter, error) {
1384 if !cfg.Logging.ContainerLogs.Enabled {
1385 log.Debug().Msg("container log collection disabled")
1386 return nil, nil
1387 }
1388
1389 // Determine log directory
1390 logDir := cfg.Logging.ContainerLogs.Dir
1391 if logDir == "" {
1392 dataDir := cfg.Server.DataDir
1393 if dataDir == "" {
1394 dataDir = DefaultDataDir()
1395 }
1396 logDir = filepath.Join(dataDir, "logs", "containers")
1397 }
1398
1399 writer, err := logwriter.New(logwriter.Config{
1400 Dir: logDir,
1401 MaxSize: cfg.Logging.ContainerLogs.MaxSize,
1402 MaxBackups: cfg.Logging.ContainerLogs.MaxBackups,
1403 MaxAge: cfg.Logging.ContainerLogs.MaxAge,
1404 })
1405 if err != nil {
1406 return nil, log.WrapErr(err, "failed to create container log writer")
1407 }
1408
1409 log.Info().Str("dir", logDir).Msg("container log collection enabled")
1410 return writer, nil
1411}
1412
1413const (
1414 internalRegistryUsername = "gordon-internal"
1415 serviceTokenSubject = "gordon-service"
1416 serviceTokenDefaultTTL = 30 * 24 * time.Hour
1417)
1418
1419func generateInternalRegistryAuth() (string, string, error) {
1420 password, err := randomTokenHex(32)
1421 if err != nil {
1422 return "", "", err
1423 }
1424 return internalRegistryUsername, password, nil
1425}
1426
1427func randomTokenHex(size int) (string, error) {
1428 buf := make([]byte, size)
1429 if _, err := rand.Read(buf); err != nil {
1430 return "", err
1431 }
1432 return hex.EncodeToString(buf), nil
1433}
1434
1435// InternalCredentials holds the internal registry credentials for CLI access.
1436type InternalCredentials struct {
1437 Username string `json:"username"`
1438 Password string `json:"password"`
1439}
1440
1441// getSecureRuntimeDir returns a secure directory for runtime files.
1442// Priority: XDG_RUNTIME_DIR > ~/.gordon/run
1443func getSecureRuntimeDir() (string, error) {
1444 // Try XDG_RUNTIME_DIR first (typically /run/user/<uid> on Linux)
1445 if runtimeDir := os.Getenv("XDG_RUNTIME_DIR"); runtimeDir != "" {
1446 gordonDir := filepath.Join(runtimeDir, "gordon")
1447 if err := os.MkdirAll(gordonDir, 0700); err == nil {
1448 return gordonDir, nil
1449 }
1450 }
1451
1452 // Fall back to ~/.gordon/run
1453 homeDir, err := os.UserHomeDir()
1454 if err != nil {
1455 return "", fmt.Errorf("failed to get home directory: %w", err)
1456 }
1457
1458 gordonDir := filepath.Join(homeDir, ".gordon", "run")
1459 if err := os.MkdirAll(gordonDir, 0700); err != nil {
1460 return "", fmt.Errorf("failed to create runtime directory: %w", err)
1461 }
1462
1463 return gordonDir, nil
1464}
1465
1466// getInternalCredentialsFile returns the path to the internal credentials file.
1467// SECURITY: Credentials are stored in a secure location with restricted permissions.
1468func getInternalCredentialsFile() string {
1469 runtimeDir, err := getSecureRuntimeDir()
1470 if err != nil {
1471 // Fall back to temp dir if we can't get secure dir (shouldn't happen)
1472 return filepath.Join(os.TempDir(), "gordon-internal-creds.json")
1473 }
1474 return filepath.Join(runtimeDir, "internal-creds.json")
1475}
1476
1477// persistInternalCredentials saves the internal registry credentials to a secure file.
1478// SECURITY: Credentials are stored in XDG_RUNTIME_DIR or ~/.gordon/run with 0600 permissions.
1479// The file is cleaned up on graceful shutdown but may persist if Gordon crashes.
1480// These credentials are for internal loopback communication only and are regenerated on each start.
1481func persistInternalCredentials(username, password string) error {
1482 creds := InternalCredentials{
1483 Username: username,
1484 Password: password,
1485 }
1486 data, err := json.Marshal(creds)
1487 if err != nil {
1488 return fmt.Errorf("failed to marshal credentials: %w", err)
1489 }
1490
1491 credFile := getInternalCredentialsFile()
1492
1493 // Ensure parent directory exists with secure permissions
1494 if err := os.MkdirAll(filepath.Dir(credFile), 0700); err != nil {
1495 return fmt.Errorf("failed to create credentials directory: %w", err)
1496 }
1497
1498 // Write file with restrictive permissions (owner read/write only)
1499 if err := os.WriteFile(credFile, data, 0600); err != nil {
1500 return fmt.Errorf("failed to write credentials file: %w", err)
1501 }
1502 return nil
1503}
1504
1505// cleanupInternalCredentials removes the internal credentials file.
1506func cleanupInternalCredentials() {
1507 _ = os.Remove(getInternalCredentialsFile())
1508}
1509
1510// getInternalCredentialsCandidates returns candidate file paths in priority order:
1511// 1. XDG_RUNTIME_DIR/gordon/ (set by systemd for the daemon)
1512// 2. /run/user/<uid>/gordon/ (well-known systemd default, for CLI in shells without XDG_RUNTIME_DIR)
1513// 3. ~/.gordon/run/ (fallback for non-systemd environments)
1514// 4. os.TempDir() (last resort, matches getInternalCredentialsFile fallback path)
1515func getInternalCredentialsCandidates() []string {
1516 var candidates []string
1517
1518 // 1. XDG_RUNTIME_DIR (set in daemon's environment)
1519 if runtimeDir := os.Getenv("XDG_RUNTIME_DIR"); runtimeDir != "" {
1520 candidates = append(candidates, filepath.Join(runtimeDir, "gordon", "internal-creds.json"))
1521 }
1522
1523 // 2. /run/user/<uid>/gordon/ (systemd default, may not be in CLI's env)
1524 uid := os.Getuid()
1525 sysRuntime := filepath.Join("/run/user", fmt.Sprintf("%d", uid), "gordon", "internal-creds.json")
1526 // Avoid duplicate if XDG_RUNTIME_DIR already points here
1527 if len(candidates) == 0 || candidates[0] != sysRuntime {
1528 candidates = append(candidates, sysRuntime)
1529 }
1530
1531 // 3. ~/.gordon/run/ fallback
1532 if homeDir, err := os.UserHomeDir(); err == nil {
1533 candidates = append(candidates, filepath.Join(homeDir, ".gordon", "run", "internal-creds.json"))
1534 }
1535
1536 // 4. os.TempDir() last resort — matches the fallback path in getInternalCredentialsFile,
1537 // ensuring GetInternalCredentials can find credentials even when getSecureRuntimeDir fails.
1538 candidates = append(candidates, filepath.Join(os.TempDir(), "gordon-internal-creds.json"))
1539
1540 return candidates
1541}
1542
1543// GetInternalCredentialsFromCandidates reads credentials from the first candidate file that exists.
1544// Exported for testing.
1545func GetInternalCredentialsFromCandidates(candidates []string) (*InternalCredentials, error) {
1546 var lastErr error
1547 for _, path := range candidates {
1548 data, err := os.ReadFile(path)
1549 if os.IsNotExist(err) {
1550 continue
1551 }
1552 if err != nil {
1553 // Non-permission errors (e.g. EACCES) may be transient or path-specific;
1554 // record and try the next candidate rather than failing immediately.
1555 lastErr = fmt.Errorf("failed to read credentials file %s: %w", path, err)
1556 continue
1557 }
1558 var creds InternalCredentials
1559 if err := json.Unmarshal(data, &creds); err != nil {
1560 // Corrupt file — record and fall through to lower-priority candidates.
1561 lastErr = fmt.Errorf("failed to parse credentials at %s: %w", path, err)
1562 continue
1563 }
1564 return &creds, nil
1565 }
1566 if lastErr != nil {
1567 return nil, lastErr
1568 }
1569 return nil, fmt.Errorf("no credentials file found (is Gordon running?): checked %v", candidates)
1570}
1571
1572// GetInternalCredentials reads the internal registry credentials from file.
1573// Probes all candidate runtime directories so CLI works regardless of whether
1574// XDG_RUNTIME_DIR is set in the current shell environment.
1575func GetInternalCredentials() (*InternalCredentials, error) {
1576 return GetInternalCredentialsFromCandidates(getInternalCredentialsCandidates())
1577}
1578
1579// createAuthService creates the authentication service and token store.
1580func createAuthService(ctx context.Context, cfg Config, log zerowrap.Logger) (out.TokenStore, *auth.Service, error) {
1581 if !cfg.Auth.Enabled {
1582 log.Warn().Msg("auth.enabled=false detected: running in local-only mode (registry loopback-only, admin API disabled)")
1583 return nil, nil, nil
1584 }
1585
1586 authType, err := resolveAuthType(cfg)
1587 if err != nil {
1588 return nil, nil, fmt.Errorf("resolve auth type: %w", err)
1589 }
1590 backend, err := resolveSecretsBackend(cfg.Auth.SecretsBackend)
1591 if err != nil {
1592 return nil, nil, log.WrapErr(err, "failed to resolve secrets backend")
1593 }
1594 dataDir := resolveDataDir(cfg.Server.DataDir)
1595
1596 store, err := createTokenStore(backend, dataDir, log)
1597 if err != nil {
1598 return nil, nil, err
1599 }
1600
1601 authConfig, err := buildAuthConfig(ctx, cfg, authType, backend, dataDir, log)
1602 if err != nil {
1603 return nil, nil, err
1604 }
1605
1606 authSvc := auth.NewService(authConfig, store, log)
1607
1608 log.Info().
1609 Str("type", string(authType)).
1610 Str("backend", string(backend)).
1611 Msg("registry authentication enabled")
1612
1613 return store, authSvc, nil
1614}
1615
1616// resolveAuthType determines the auth type from config.
1617// Token-only authentication is the only supported mode.
1618func resolveAuthType(cfg Config) (domain.AuthType, error) {
1619 if cfg.Auth.Type != "" && cfg.Auth.Type != "token" {
1620 return "", fmt.Errorf("unsupported auth.type %q; only \"token\" is supported", cfg.Auth.Type)
1621 }
1622 return domain.AuthTypeToken, nil
1623}
1624
1625func resolveSecretsBackend(backend string) (domain.SecretsBackend, error) {
1626 switch backend {
1627 case "pass":
1628 return domain.SecretsBackendPass, nil
1629 case "sops":
1630 return domain.SecretsBackendSops, nil
1631 case "unsafe":
1632 return domain.SecretsBackendUnsafe, nil
1633 case "":
1634 return "", fmt.Errorf("auth.secrets_backend is required")
1635 default:
1636 return "", fmt.Errorf("unsupported auth.secrets_backend %q", backend)
1637 }
1638}
1639
1640func resolveDataDir(dataDir string) string {
1641 if dataDir == "" {
1642 return DefaultDataDir()
1643 }
1644 return dataDir
1645}
1646
1647func resolveEnvDir(cfg Config) string {
1648 dataDir := resolveDataDir(cfg.Server.DataDir)
1649 envDir := cfg.Env.Dir
1650 if envDir == "" {
1651 envDir = filepath.Join(dataDir, "env")
1652 }
1653 return envDir
1654}
1655
1656func resolveRegistryDomains(cfg Config) (string, []string) {
1657 registryDomain := cfg.Server.RegistryDomain
1658 if registryDomain == "" {
1659 registryDomain = cfg.Server.GordonDomain
1660 }
1661 return registryDomain, append([]string{}, cfg.Server.LegacyRegistryDomains...)
1662}
1663
1664func createTokenStore(backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) (out.TokenStore, error) {
1665 // Token store is always created since tokens work in both auth modes
1666 store, err := tokenstore.NewStore(backend, dataDir, log)
1667 if err != nil {
1668 return nil, log.WrapErr(err, "failed to create token store")
1669 }
1670 return store, nil
1671}
1672
1673func buildAuthConfig(ctx context.Context, cfg Config, authType domain.AuthType, backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) (auth.Config, error) {
1674 authConfig := auth.Config{
1675 Enabled: cfg.Auth.Enabled,
1676 AuthType: authType,
1677 Username: cfg.Auth.Username,
1678 }
1679
1680 // Token config is always required (tokens work in all auth modes)
1681 secret, expiry, err := loadTokenConfig(ctx, cfg, backend, dataDir, log)
1682 if err != nil {
1683 return auth.Config{}, err
1684 }
1685 authConfig.TokenSecret = secret
1686 authConfig.TokenExpiry = expiry
1687
1688 accessTokenTTL := 15 * time.Minute // default
1689 if cfg.Auth.AccessTokenTTL != "" {
1690 parsed, err := time.ParseDuration(cfg.Auth.AccessTokenTTL)
1691 if err != nil {
1692 return auth.Config{}, fmt.Errorf("invalid auth.access_token_ttl %q: %w", cfg.Auth.AccessTokenTTL, err)
1693 }
1694 if parsed <= 0 {
1695 return auth.Config{}, fmt.Errorf("auth.access_token_ttl must be positive")
1696 }
1697 if parsed > auth.MaxAccessTokenLifetime {
1698 return auth.Config{}, fmt.Errorf("auth.access_token_ttl must not exceed %v", auth.MaxAccessTokenLifetime)
1699 }
1700 accessTokenTTL = parsed
1701 }
1702 authConfig.AccessTokenTTL = accessTokenTTL
1703
1704 return authConfig, nil
1705}
1706
1707func loadTokenConfig(ctx context.Context, cfg Config, backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) ([]byte, time.Duration, error) {
1708 secret, err := loadTokenSecret(ctx, cfg, backend, dataDir, log)
1709 if err != nil {
1710 return nil, 0, err
1711 }
1712
1713 expiry, err := parseTokenExpiry(cfg.Auth.TokenExpiry)
1714 if err != nil {
1715 return nil, 0, err
1716 }
1717
1718 return secret, expiry, nil
1719}
1720
1721// TokenSecretEnvVar is the environment variable for the JWT signing secret.
1722// SECURITY: This takes priority over config file to allow secure secret injection.
1723const TokenSecretEnvVar = "GORDON_AUTH_TOKEN_SECRET" //nolint:gosec // This is an env var name, not a credential
1724
1725func loadTokenSecret(ctx context.Context, cfg Config, backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) ([]byte, error) {
1726 // SECURITY: Priority order for token secret:
1727 // 1. Environment variable (most secure - no disk exposure)
1728 // 2. Secrets backend (pass/sops - encrypted)
1729 // 3. Config file path (least preferred)
1730
1731 const minTokenSecretLength = 32
1732
1733 // Check environment variable first
1734 if envSecret := os.Getenv(TokenSecretEnvVar); envSecret != "" {
1735 if len(envSecret) < minTokenSecretLength {
1736 return nil, fmt.Errorf("token secret from %s must be at least %d bytes (got %d)", TokenSecretEnvVar, minTokenSecretLength, len(envSecret))
1737 }
1738 log.Debug().Msg("using token secret from environment variable")
1739 return []byte(envSecret), nil
1740 }
1741
1742 // Fall back to config-specified path via secrets backend
1743 if cfg.Auth.TokenSecret == "" {
1744 return nil, fmt.Errorf("token_secret is required for JWT token generation; set %s environment variable or configure auth.token_secret", TokenSecretEnvVar)
1745 }
1746
1747 secret, err := loadSecret(ctx, backend, cfg.Auth.TokenSecret, dataDir, log)
1748 if err != nil {
1749 return nil, log.WrapErr(err, "failed to load token secret")
1750 }
1751
1752 if len(secret) < minTokenSecretLength {
1753 return nil, fmt.Errorf("token_secret must be at least %d bytes (got %d); use a strong random secret", minTokenSecretLength, len(secret))
1754 }
1755
1756 return []byte(secret), nil
1757}
1758
1759func parseTokenExpiry(expiry string) (time.Duration, error) {
1760 if expiry == "" {
1761 return 0, nil
1762 }
1763
1764 parsed, err := duration.Parse(expiry)
1765 if err != nil {
1766 return 0, fmt.Errorf("invalid token_expiry: %w", err)
1767 }
1768
1769 return parsed, nil
1770}
1771
1772// loadSecret loads a secret from the configured backend.
1773func loadSecret(ctx context.Context, backend domain.SecretsBackend, path, dataDir string, log zerowrap.Logger) (string, error) {
1774 switch backend {
1775 case domain.SecretsBackendPass:
1776 provider := secrets.NewPassProvider(log)
1777 return provider.GetSecret(ctx, path)
1778 case domain.SecretsBackendSops:
1779 provider := secrets.NewSopsProvider(log)
1780 return provider.GetSecret(ctx, path)
1781 case domain.SecretsBackendUnsafe:
1782 // For unsafe backend, path is relative to dataDir/secrets/.
1783 return readUnsafeSecret(dataDir, path)
1784 default:
1785 return "", fmt.Errorf("unknown secrets backend: %s", backend)
1786 }
1787}
1788
1789func readFileBeneath(root, cleanedRelPath string) ([]byte, error) {
1790 rootFD, err := unix.Open(root, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
1791 if err != nil {
1792 return nil, fmt.Errorf("failed to open secrets root: %w", err)
1793 }
1794 defer unix.Close(rootFD)
1795
1796 parts := strings.Split(filepath.ToSlash(cleanedRelPath), "/")
1797 dirFD := rootFD
1798 var closeDirFDs []int
1799 defer func() {
1800 for i := len(closeDirFDs) - 1; i >= 0; i-- {
1801 _ = unix.Close(closeDirFDs[i])
1802 }
1803 }()
1804
1805 for i, part := range parts {
1806 if part == "" || part == "." || part == ".." {
1807 return nil, fmt.Errorf("invalid secret path: path must stay under dataDir/secrets")
1808 }
1809 last := i == len(parts)-1
1810 if last {
1811 fd, err := unix.Openat(dirFD, part, unix.O_RDONLY|unix.O_NONBLOCK|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
1812 if err != nil {
1813 return nil, fmt.Errorf("failed to read secret file: %w", err)
1814 }
1815 defer unix.Close(fd)
1816 var st unix.Stat_t
1817 if err := unix.Fstat(fd, &st); err != nil {
1818 return nil, fmt.Errorf("failed to stat secret file: %w", err)
1819 }
1820 if st.Mode&unix.S_IFMT != unix.S_IFREG {
1821 return nil, fmt.Errorf("invalid secret path: secret must be a regular file")
1822 }
1823 data, err := os.ReadFile(fmt.Sprintf("/proc/self/fd/%d", fd))
1824 if err != nil {
1825 return nil, fmt.Errorf("failed to read secret file: %w", err)
1826 }
1827 return data, nil
1828 }
1829
1830 nextFD, err := unix.Openat(dirFD, part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
1831 if err != nil {
1832 return nil, fmt.Errorf("failed to open secret path component: %w", err)
1833 }
1834 closeDirFDs = append(closeDirFDs, nextFD)
1835 dirFD = nextFD
1836 }
1837
1838 return nil, fmt.Errorf("invalid secret path: empty path")
1839}
1840
1841type unsafeSecretProvider struct {
1842 dataDir string
1843}
1844
1845func (p unsafeSecretProvider) Name() string { return string(domain.SecretsBackendUnsafe) }
1846
1847func (p unsafeSecretProvider) IsAvailable() bool { return true }
1848
1849func (p unsafeSecretProvider) GetSecret(_ context.Context, path string) (string, error) {
1850 return readUnsafeSecret(p.dataDir, path)
1851}
1852
1853func createStandaloneServiceSecretProvider(backend domain.SecretsBackend, dataDir string, log zerowrap.Logger) out.SecretProvider {
1854 switch backend {
1855 case domain.SecretsBackendPass:
1856 return secrets.NewPassProvider(log)
1857 case domain.SecretsBackendSops:
1858 return secrets.NewSopsProvider(log)
1859 case domain.SecretsBackendUnsafe:
1860 return unsafeSecretProvider{dataDir: dataDir}
1861 default:
1862 return nil
1863 }
1864}
1865
1866func readUnsafeSecret(dataDir, secretPath string) (string, error) {
1867 if filepath.IsAbs(secretPath) {
1868 return "", fmt.Errorf("invalid secret path: absolute paths are not allowed")
1869 }
1870 cleaned := filepath.Clean(secretPath)
1871 if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) {
1872 return "", fmt.Errorf("invalid secret path: path must stay under dataDir/secrets")
1873 }
1874
1875 root := filepath.Clean(filepath.Join(dataDir, "secrets"))
1876 data, err := readFileBeneath(root, cleaned)
1877 if err != nil {
1878 return "", err
1879 }
1880 return string(data), nil
1881}
1882
1883// proxyConfigResult holds parsed proxy and blob chunk size config.
1884type proxyConfigResult struct {
1885 proxyConfig proxy.Config
1886 maxBlobChunkSize int64
1887 maxBlobSize int64
1888}
1889
1890type configWatcher interface {
1891 Watch(ctx context.Context, onChange func()) error
1892}
1893
1894// publicTLSReconciler is the interface for reconciling public TLS certificates.
1895type publicTLSReconciler interface {
1896 Reconcile(context.Context) error
1897}
1898
1899type configReloader interface {
1900 Reload(ctx context.Context) error
1901}
1902
1903type proxyConfigUpdater interface {
1904 UpdateConfig(config proxy.Config)
1905}
1906
1907type reloadTrigger interface {
1908 Trigger(ctx context.Context) error
1909}
1910
1911type loadedConfigApplier interface {
1912 ApplyLoadedConfig(ctx context.Context) error
1913}
1914
1915type reloadCoordinator struct {
1916 mu sync.Mutex
1917 lastRun time.Time
1918 debounce time.Duration
1919 trailingTimer *time.Timer
1920 trailingGeneration uint64
1921 stopped bool
1922
1923 // lifecycleCtx owns the context used by debounced trailing reloads. Its
1924 // base is detached from any caller's cancellation so a short-lived request
1925 // context cannot abort a coalesced apply, while lifecycleCancel lets Stop
1926 // tear the coordinator's own lifecycle down explicitly.
1927 lifecycleCtx context.Context
1928 lifecycleCancel context.CancelFunc
1929
1930 configSvc configReloader
1931 v *viper.Viper
1932 proxySvc proxyConfigUpdater
1933 applyContainerConfig func(context.Context, Config) error
1934 applyAppMountPolicy func(context.Context, Config) error
1935 registryLimits interface {
1936 UpdateBlobLimits(maxBlobChunkSize, maxBlobSize int64)
1937 }
1938 eventBus out.EventPublisher
1939 publicTLS publicTLSReconciler
1940 log zerowrap.Logger
1941}
1942
1943func newReloadCoordinator(v *viper.Viper, configSvc configReloader, proxySvc proxyConfigUpdater, registryLimits interface {
1944 UpdateBlobLimits(maxBlobChunkSize, maxBlobSize int64)
1945}, eventBus out.EventPublisher, publicTLS publicTLSReconciler, log zerowrap.Logger) *reloadCoordinator {
1946 return &reloadCoordinator{
1947 debounce: 500 * time.Millisecond,
1948 configSvc: configSvc,
1949 v: v,
1950 proxySvc: proxySvc,
1951 registryLimits: registryLimits,
1952 eventBus: eventBus,
1953 publicTLS: publicTLS,
1954 log: log,
1955 }
1956}
1957
1958func (c *reloadCoordinator) SetRegistryLimits(limits interface {
1959 UpdateBlobLimits(maxBlobChunkSize, maxBlobSize int64)
1960}) {
1961 c.mu.Lock()
1962 defer c.mu.Unlock()
1963 c.registryLimits = limits
1964}
1965
1966func (c *reloadCoordinator) SetContainerConfigApplier(apply func(context.Context, Config) error) {
1967 c.mu.Lock()
1968 defer c.mu.Unlock()
1969 c.applyContainerConfig = apply
1970}
1971
1972// SetAppMountPoliciesApplier wires the callback that atomically republishes
1973// validated administrative bind policies after a successful reload.
1974func (c *reloadCoordinator) SetAppMountPoliciesApplier(apply func(context.Context, Config) error) {
1975 c.mu.Lock()
1976 defer c.mu.Unlock()
1977 c.applyAppMountPolicy = apply
1978}
1979
1980// Trigger requests a config reload that first re-reads config from disk.
1981func (c *reloadCoordinator) Trigger(ctx context.Context) error {
1982 c.mu.Lock()
1983 defer c.mu.Unlock()
1984
1985 return c.reloadDebouncedLocked(ctx, true)
1986}
1987
1988// ApplyLoadedConfig requests a reload of the config the watcher already
1989// loaded from disk. It shares the debounce/coalescing policy with Trigger so
1990// a burst of fsnotify callbacks applies the final state exactly once.
1991func (c *reloadCoordinator) ApplyLoadedConfig(ctx context.Context) error {
1992 c.mu.Lock()
1993 defer c.mu.Unlock()
1994
1995 return c.reloadDebouncedLocked(ctx, false)
1996}
1997
1998// reloadDebouncedLocked is the single owner of the debounce/coalescing policy
1999// shared by every reload entrypoint. Requests inside the debounce window are
2000// merged into one trailing reload carrying the latest loadConfig intent.
2001func (c *reloadCoordinator) reloadDebouncedLocked(ctx context.Context, loadConfig bool) error {
2002 now := time.Now()
2003 if !c.lastRun.IsZero() && now.Sub(c.lastRun) < c.debounce {
2004 c.scheduleTrailingReloadLocked(ctx, loadConfig)
2005 return nil
2006 }
2007
2008 return c.reloadLocked(ctx, loadConfig)
2009}
2010
2011func (c *reloadCoordinator) scheduleTrailingReloadLocked(ctx context.Context, loadConfig bool) {
2012 if c.stopped {
2013 return
2014 }
2015 if c.trailingTimer != nil {
2016 c.trailingTimer.Stop()
2017 }
2018 if c.lifecycleCancel == nil {
2019 c.lifecycleCtx, c.lifecycleCancel = context.WithCancel(context.WithoutCancel(ctx))
2020 }
2021 c.trailingGeneration++
2022 generation := c.trailingGeneration
2023 trailingCtx := c.lifecycleCtx
2024 c.trailingTimer = time.AfterFunc(c.debounce, func() {
2025 c.runTrailingReload(trailingCtx, generation, loadConfig)
2026 })
2027 c.log.Debug().Msg("coalescing config reload trigger")
2028}
2029
2030func (c *reloadCoordinator) runTrailingReload(ctx context.Context, generation uint64, loadConfig bool) {
2031 c.mu.Lock()
2032 defer c.mu.Unlock()
2033
2034 if c.stopped || generation != c.trailingGeneration {
2035 return
2036 }
2037 c.trailingTimer = nil
2038 if err := c.reloadLocked(ctx, loadConfig); err != nil {
2039 c.log.Error().Err(err).Msg("failed trailing config reload")
2040 }
2041}
2042
2043func (c *reloadCoordinator) Stop() {
2044 c.mu.Lock()
2045 defer c.mu.Unlock()
2046 c.stopped = true
2047 c.trailingGeneration++
2048 if c.trailingTimer != nil {
2049 c.trailingTimer.Stop()
2050 c.trailingTimer = nil
2051 }
2052 if c.lifecycleCancel != nil {
2053 c.lifecycleCancel()
2054 }
2055}
2056
2057func (c *reloadCoordinator) reloadLocked(ctx context.Context, loadConfig bool) error {
2058 now := time.Now()
2059 if loadConfig {
2060 if err := c.configSvc.Reload(ctx); err != nil {
2061 c.log.Error().Err(err).Msg("failed to reload config")
2062 return fmt.Errorf("failed to reload config: %w", err)
2063 }
2064 }
2065
2066 if err := c.applyLoadedConfig(ctx, now); err != nil {
2067 return err
2068 }
2069
2070 return nil
2071}
2072
2073func (c *reloadCoordinator) applyLoadedConfig(ctx context.Context, now time.Time) error {
2074 var reloadCfg Config
2075 if err := c.v.Unmarshal(&reloadCfg); err != nil {
2076 c.log.Error().Err(err).Msg("failed to unmarshal config on reload")
2077 return fmt.Errorf("failed to unmarshal config on reload: %w", err)
2078 }
2079 if err := validateEntrypointMigration(c.v, reloadCfg); err != nil {
2080 return err
2081 }
2082 if err := validateRetiredAppConfig(c.v); err != nil {
2083 return err
2084 }
2085 if err := validateAppPolicies(c.v, reloadCfg); err != nil {
2086 return err
2087 }
2088
2089 reloadedProxy, err := buildProxyConfig(reloadCfg, c.log)
2090 if err != nil {
2091 c.log.Error().Err(err).Msg("failed to parse proxy config on reload")
2092 return fmt.Errorf("failed to parse proxy config on reload: %w", err)
2093 }
2094
2095 if c.applyContainerConfig != nil {
2096 if err := c.applyContainerConfig(ctx, reloadCfg); err != nil {
2097 c.log.Error().Err(err).Msg("failed to apply container config on reload")
2098 return fmt.Errorf("failed to apply container config on reload: %w", err)
2099 }
2100 }
2101 // Publish the reloaded bind policies only after conversion and shape
2102 // validation succeed, so a bad edit keeps the previous policies live.
2103 if c.applyAppMountPolicy != nil {
2104 if err := c.applyAppMountPolicy(ctx, reloadCfg); err != nil {
2105 c.log.Error().Err(err).Msg("failed to apply app mount policies on reload")
2106 return fmt.Errorf("failed to apply app mount policies on reload: %w", err)
2107 }
2108 }
2109 c.proxySvc.UpdateConfig(reloadedProxy.proxyConfig)
2110 if c.registryLimits != nil {
2111 c.registryLimits.UpdateBlobLimits(reloadedProxy.maxBlobChunkSize, reloadedProxy.maxBlobSize)
2112 }
2113
2114 // Reconcile public TLS before publishing reload events so certificate
2115 // authorization reflects the loaded config even if event delivery fails.
2116 // A transient ACME issue must not abort the rest of the reload.
2117 if c.publicTLS != nil {
2118 if err := c.publicTLS.Reconcile(ctx); err != nil {
2119 c.log.Warn().Err(err).Msg("failed to reconcile public TLS certificates after reload, continuing")
2120 }
2121 }
2122
2123 if c.eventBus != nil {
2124 if err := c.eventBus.Publish(domain.EventConfigReload, nil); err != nil {
2125 c.log.Error().Err(err).Msg("failed to publish config reload event")
2126 return fmt.Errorf("failed to publish config reload event: %w", err)
2127 }
2128 }
2129
2130 c.lastRun = now
2131
2132 c.log.Debug().Msg("config hot reload complete")
2133 return nil
2134}
2135
2136// buildProxyConfig parses size-related config fields and builds the proxy config.
2137func buildProxyConfig(cfg Config, log zerowrap.Logger) (*proxyConfigResult, error) {
2138 maxProxyBodySize := int64(512 << 20) // 512MB default
2139 if cfg.Server.MaxProxyBodySize != "" {
2140 parsedSize, err := bytesize.Parse(cfg.Server.MaxProxyBodySize)
2141 if err != nil {
2142 return nil, log.WrapErrWithFields(err, "invalid server.max_proxy_body_size configuration", map[string]any{"value": cfg.Server.MaxProxyBodySize})
2143 }
2144 maxProxyBodySize = parsedSize
2145 }
2146
2147 maxBlobChunkSize := int64(registry.DefaultMaxBlobChunkSize)
2148 if cfg.Server.MaxBlobChunkSize != "" {
2149 parsedSize, err := bytesize.Parse(cfg.Server.MaxBlobChunkSize)
2150 if err != nil {
2151 return nil, log.WrapErrWithFields(err, "invalid server.max_blob_chunk_size configuration", map[string]any{"value": cfg.Server.MaxBlobChunkSize})
2152 }
2153 maxBlobChunkSize = parsedSize
2154 }
2155
2156 maxBlobSize := int64(registry.DefaultMaxBlobSize)
2157 if cfg.Server.MaxBlobSize != "" {
2158 parsedSize, err := bytesize.Parse(cfg.Server.MaxBlobSize)
2159 if err != nil {
2160 return nil, log.WrapErrWithFields(err, "invalid server.max_blob_size configuration", map[string]any{"value": cfg.Server.MaxBlobSize})
2161 }
2162 maxBlobSize = parsedSize
2163 }
2164
2165 maxProxyResponseSize := int64(1 << 30) // 1GB default
2166 if cfg.Server.MaxProxyResponseSize != "" {
2167 parsedSize, err := bytesize.Parse(cfg.Server.MaxProxyResponseSize)
2168 if err != nil {
2169 return nil, log.WrapErrWithFields(err, "invalid server.max_proxy_response_size configuration", map[string]any{"value": cfg.Server.MaxProxyResponseSize})
2170 }
2171 maxProxyResponseSize = parsedSize
2172 }
2173
2174 maxConcurrentConns := cfg.Server.MaxConcurrentConns
2175 if maxConcurrentConns < 0 {
2176 maxConcurrentConns = 10000 // default when explicitly set to -1
2177 }
2178 // 0 means no limit (as documented in proxy.Config)
2179
2180 registryDomain, _ := resolveRegistryDomains(cfg)
2181
2182 return &proxyConfigResult{
2183 proxyConfig: proxy.Config{
2184 RegistryDomain: registryDomain,
2185 RegistryPort: cfg.Server.RegistryPort,
2186 MaxBodySize: maxProxyBodySize,
2187 MaxResponseSize: maxProxyResponseSize,
2188 MaxConcurrentConns: maxConcurrentConns,
2189 },
2190 maxBlobChunkSize: maxBlobChunkSize,
2191 maxBlobSize: maxBlobSize,
2192 }, nil
2193}
2194
2195// buildDNSConfig parses the raw dns config section into a publictls.DNSConfig.
2196func buildDNSConfig(cfg Config) (publictls.DNSConfig, error) {
2197 defaults := publictls.DefaultDNSConfig()
2198
2199 resolvers := cfg.DNS.Resolvers
2200 if len(resolvers) == 0 {
2201 resolvers = defaults.Resolvers
2202 }
2203
2204 propagationTimeout := defaults.PropagationTimeout
2205 if cfg.DNS.PropagationTimeout != "" {
2206 parsed, err := time.ParseDuration(cfg.DNS.PropagationTimeout)
2207 if err != nil {
2208 return publictls.DNSConfig{}, fmt.Errorf("invalid dns.propagation_timeout: %w", err)
2209 }
2210 propagationTimeout = parsed
2211 }
2212
2213 pollingInterval := defaults.PollingInterval
2214 if cfg.DNS.PollingInterval != "" {
2215 parsed, err := time.ParseDuration(cfg.DNS.PollingInterval)
2216 if err != nil {
2217 return publictls.DNSConfig{}, fmt.Errorf("invalid dns.polling_interval: %w", err)
2218 }
2219 pollingInterval = parsed
2220 }
2221
2222 dnsCfg := publictls.DNSConfig{
2223 Resolvers: append([]string(nil), resolvers...),
2224 PropagationTimeout: propagationTimeout,
2225 PollingInterval: pollingInterval,
2226 }
2227 if err := dnsCfg.Validate(); err != nil {
2228 return publictls.DNSConfig{}, err
2229 }
2230 return dnsCfg, nil
2231}
2232
2233func buildContainerServiceConfig(_ context.Context, v *viper.Viper, _ Config, _ *services, _ zerowrap.Logger) (container.Config, error) {
2234 return container.Config{
2235 NetworkPrefix: v.GetString("network_isolation.network_prefix"),
2236 }, nil
2237}
2238
2239// createContainerService creates the container service with configuration.
2240func createContainerService(ctx context.Context, v *viper.Viper, cfg Config, svc *services, log zerowrap.Logger) (*container.Service, error) {
2241 containerConfig, err := buildContainerServiceConfig(ctx, v, cfg, svc, log)
2242 if err != nil {
2243 return nil, err
2244 }
2245 return container.NewService(svc.runtime, svc.envLoader, svc.eventBus, svc.logWriter, containerConfig), nil
2246}
2247
2248type databaseBackupSettingsConfig struct {
2249 Enabled bool
2250 Schedule string
2251 StorageDir string
2252 Retention struct {
2253 Hourly int
2254 Daily int
2255 Weekly int
2256 Monthly int
2257 }
2258}
2259
2260func databaseBackupSettings(cfg Config) databaseBackupSettingsConfig {
2261 out := databaseBackupSettingsConfig{
2262 Enabled: cfg.Backups.Databases.Enabled,
2263 Schedule: cfg.Backups.Databases.Schedule,
2264 StorageDir: cfg.Backups.Databases.StorageDir,
2265 }
2266 out.Retention.Hourly = cfg.Backups.Databases.Retention.Hourly
2267 out.Retention.Daily = cfg.Backups.Databases.Retention.Daily
2268 out.Retention.Weekly = cfg.Backups.Databases.Retention.Weekly
2269 out.Retention.Monthly = cfg.Backups.Databases.Retention.Monthly
2270 // Legacy [backups] keys intentionally override new database defaults when
2271 // backups.enabled is true, preserving existing working pg_dump schedules.
2272 // Otherwise, prefer the already-populated backups.databases.* values.
2273 if cfg.Backups.Enabled {
2274 out.Enabled = true
2275 if cfg.Backups.Schedule != "" {
2276 out.Schedule = cfg.Backups.Schedule
2277 }
2278 if cfg.Backups.StorageDir != "" {
2279 out.StorageDir = cfg.Backups.StorageDir
2280 }
2281 } else {
2282 if out.Schedule == "" {
2283 out.Schedule = cfg.Backups.Schedule
2284 }
2285 if out.StorageDir == "" {
2286 out.StorageDir = cfg.Backups.StorageDir
2287 }
2288 }
2289 if out.Retention.Hourly == 0 {
2290 out.Retention.Hourly = cfg.Backups.Retention.Hourly
2291 }
2292 if out.Retention.Daily == 0 {
2293 out.Retention.Daily = cfg.Backups.Retention.Daily
2294 }
2295 if out.Retention.Weekly == 0 {
2296 out.Retention.Weekly = cfg.Backups.Retention.Weekly
2297 }
2298 if out.Retention.Monthly == 0 {
2299 out.Retention.Monthly = cfg.Backups.Retention.Monthly
2300 }
2301 return out
2302}
2303
2304func createBackupService(cfg Config, svc *services, log zerowrap.Logger) (*filesystem.BackupStorage, *backup.Service, error) {
2305 dbCfg := databaseBackupSettings(cfg)
2306 if !dbCfg.Enabled {
2307 return nil, nil, nil
2308 }
2309
2310 storageDir := dbCfg.StorageDir
2311 if storageDir == "" {
2312 dataDir := resolveDataDir(cfg.Server.DataDir)
2313 storageDir = filepath.Join(dataDir, "backups")
2314 }
2315
2316 backupStorage, err := filesystem.NewBackupStorage(storageDir, log)
2317 if err != nil {
2318 return nil, nil, log.WrapErr(err, "failed to create backup storage")
2319 }
2320
2321 retention, err := validateBackupRetention(cfg)
2322 if err != nil {
2323 return nil, nil, log.WrapErr(err, "invalid backup retention policy")
2324 }
2325
2326 backupCfg := domain.BackupConfig{
2327 Enabled: dbCfg.Enabled,
2328 StorageDir: storageDir,
2329 Retention: retention,
2330 }
2331
2332 backupSvc := backup.NewService(svc.runtime, backupStorage, backupCfg, log)
2333
2334 log.Info().
2335 Str("storage_dir", storageDir).
2336 Msg("backup service initialized")
2337
2338 return backupStorage, backupSvc, nil
2339}
2340
2341func createVolumeBackupService(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger) (out.VolumeBackupStorage, *backup.VolumeService, domain.VolumeBackupConfig, error) {
2342 if !cfg.Backups.Volumes.Enabled {
2343 return nil, nil, domain.VolumeBackupConfig{}, nil
2344 }
2345 volumeCfg, err := validateVolumeBackupConfig(cfg)
2346 if err != nil {
2347 return nil, nil, domain.VolumeBackupConfig{}, log.WrapErr(err, "invalid volume backup configuration")
2348 }
2349
2350 storage, err := s3storage.NewVolumeBackupStorage(ctx, volumeCfg)
2351 if err != nil {
2352 return nil, nil, domain.VolumeBackupConfig{}, log.WrapErr(err, "failed to create volume backup storage")
2353 }
2354
2355 volumeSvc := backup.NewVolumeService(svc.runtime, storage, volumeCfg, log)
2356 log.Info().
2357 Str("bucket", volumeCfg.S3Bucket).
2358 Str("prefix", volumeCfg.S3Prefix).
2359 Msg("volume backup service initialized")
2360
2361 return storage, volumeSvc, volumeCfg, nil
2362}
2363
2364func validateBackupRetention(cfg Config) (domain.RetentionPolicy, error) {
2365 dbCfg := databaseBackupSettings(cfg)
2366 if dbCfg.Retention.Hourly < 0 {
2367 return domain.RetentionPolicy{}, fmt.Errorf("backups.databases.retention.hourly cannot be negative")
2368 }
2369 if dbCfg.Retention.Daily < 0 {
2370 return domain.RetentionPolicy{}, fmt.Errorf("backups.databases.retention.daily cannot be negative")
2371 }
2372 if dbCfg.Retention.Weekly < 0 {
2373 return domain.RetentionPolicy{}, fmt.Errorf("backups.databases.retention.weekly cannot be negative")
2374 }
2375 if dbCfg.Retention.Monthly < 0 {
2376 return domain.RetentionPolicy{}, fmt.Errorf("backups.databases.retention.monthly cannot be negative")
2377 }
2378
2379 return domain.RetentionPolicy{
2380 Hourly: dbCfg.Retention.Hourly,
2381 Daily: dbCfg.Retention.Daily,
2382 Weekly: dbCfg.Retention.Weekly,
2383 Monthly: dbCfg.Retention.Monthly,
2384 }, nil
2385}
2386
2387func validateVolumeBackupConfig(cfg Config) (domain.VolumeBackupConfig, error) {
2388 volumeCfg := cfg.Backups.Volumes
2389 interval, err := parsePositiveDurationDefault(volumeCfg.Interval, "24h", "backups.volumes.interval")
2390 if err != nil {
2391 return domain.VolumeBackupConfig{}, err
2392 }
2393 timeout, err := parsePositiveDurationDefault(volumeCfg.Timeout, "2h", "backups.volumes.timeout")
2394 if err != nil {
2395 return domain.VolumeBackupConfig{}, err
2396 }
2397 compression, err := parseVolumeBackupCompression(volumeCfg.Compression)
2398 if err != nil {
2399 return domain.VolumeBackupConfig{}, err
2400 }
2401 maxConcurrency := volumeCfg.MaxConcurrency
2402 if maxConcurrency == 0 {
2403 maxConcurrency = 2
2404 }
2405 helperImage := strings.TrimSpace(volumeCfg.HelperImage)
2406 if helperImage == "" {
2407 helperImage = "alpine:3.20"
2408 }
2409 volumePrefix := strings.TrimSpace(cfg.Volumes.Prefix)
2410 if volumePrefix == "" {
2411 volumePrefix = "gordon"
2412 }
2413 if compression == domain.VolumeBackupCompressionZstd && helperImage == "alpine:3.20" {
2414 return domain.VolumeBackupConfig{}, fmt.Errorf("backups.volumes.compression zstd requires a helper_image that provides zstd")
2415 }
2416 if err := validateVolumeBackupS3Settings(volumeCfg.Enabled, volumeCfg.Retention.Keep, maxConcurrency, volumeCfg.S3.Bucket, volumeCfg.S3.Region); err != nil {
2417 return domain.VolumeBackupConfig{}, err
2418 }
2419 return domain.VolumeBackupConfig{
2420 Enabled: volumeCfg.Enabled,
2421 Interval: interval,
2422 Compression: compression,
2423 Retention: domain.VolumeBackupRetentionPolicy{Keep: volumeCfg.Retention.Keep},
2424 Timeout: timeout,
2425 MaxConcurrency: maxConcurrency,
2426 HelperImage: helperImage,
2427 VolumePrefix: volumePrefix,
2428 S3Bucket: strings.TrimSpace(volumeCfg.S3.Bucket),
2429 S3Region: strings.TrimSpace(volumeCfg.S3.Region),
2430 S3Prefix: strings.TrimSpace(volumeCfg.S3.Prefix),
2431 S3Endpoint: strings.TrimSpace(volumeCfg.S3.Endpoint),
2432 S3PathStyle: volumeCfg.S3.PathStyle,
2433 S3SSEAlgorithm: strings.TrimSpace(volumeCfg.S3.SSEAlgorithm),
2434 S3SSEKMSKeyID: strings.TrimSpace(volumeCfg.S3.SSEKMSKeyID),
2435 }, nil
2436}
2437
2438func parsePositiveDurationDefault(raw, defaultValue, field string) (time.Duration, error) {
2439 if strings.TrimSpace(raw) == "" {
2440 raw = defaultValue
2441 }
2442 d, err := time.ParseDuration(strings.TrimSpace(raw))
2443 if err != nil || d <= 0 {
2444 return 0, fmt.Errorf("%s must be a positive duration", field)
2445 }
2446 return d, nil
2447}
2448
2449func parseVolumeBackupCompression(raw string) (domain.VolumeBackupCompression, error) {
2450 if strings.TrimSpace(raw) == "" {
2451 raw = string(domain.VolumeBackupCompressionGzip)
2452 }
2453 compression := domain.VolumeBackupCompression(strings.ToLower(strings.TrimSpace(raw)))
2454 switch compression {
2455 case domain.VolumeBackupCompressionGzip, domain.VolumeBackupCompressionZstd:
2456 return compression, nil
2457 default:
2458 return "", fmt.Errorf("backups.volumes.compression must be one of: gzip, zstd")
2459 }
2460}
2461
2462func validateVolumeBackupS3Settings(enabled bool, keep, maxConcurrency int, bucket, region string) error {
2463 if keep < 0 {
2464 return fmt.Errorf("backups.volumes.retention.keep cannot be negative")
2465 }
2466 if enabled && keep == 0 {
2467 return fmt.Errorf("backups.volumes.retention.keep must be positive when volume backups are enabled")
2468 }
2469 if maxConcurrency < 1 {
2470 return fmt.Errorf("backups.volumes.max_concurrency must be at least 1")
2471 }
2472 if enabled && strings.TrimSpace(bucket) == "" {
2473 return fmt.Errorf("backups.volumes.s3.bucket is required when volume backups are enabled")
2474 }
2475 if enabled && strings.TrimSpace(region) == "" {
2476 return fmt.Errorf("backups.volumes.s3.region is required when volume backups are enabled")
2477 }
2478 return nil
2479}
2480
2481// registerEventHandlers registers event handlers. Push-triggered
2482// route creation (auto-route, previews, image-pushed deploy) is
2483// retired: pushes transfer OCI content only and apps deploy
2484// explicitly via `gordon apps deploy`. The pre-v3 route-engine
2485// deploy arms (config-reload redeploy, manual SIGUSR2 deploy,
2486// secrets-changed redeploy) are removed with the declarative-apps
2487// cutover: reload never activates app state, and app secret changes
2488// take effect on explicit deploy.
2489func registerEventHandlers(ctx context.Context, svc *services) (func(), error) {
2490 // Proxy cache invalidation on config reload (clears stale targets for removed routes)
2491 configReloadProxyHandler := proxy.NewConfigReloadProxyHandler(ctx, svc.proxySvc)
2492 if err := svc.eventBus.Subscribe(configReloadProxyHandler); err != nil {
2493 return nil, fmt.Errorf("failed to subscribe config reload proxy handler: %w", err)
2494 }
2495
2496 cleanup := func() {
2497 if svc.reloadCoordinator != nil {
2498 svc.reloadCoordinator.Stop()
2499 }
2500 }
2501
2502 return cleanup, nil
2503}
2504
2505// setupConfigHotReload sets up config hot reload.
2506func setupConfigHotReload(ctx context.Context, configSvc configWatcher, coordinator loadedConfigApplier) error {
2507 if err := configSvc.Watch(ctx, func() {
2508 _ = coordinator.ApplyLoadedConfig(ctx)
2509 }); err != nil {
2510 return fmt.Errorf("failed to watch config: %w", err)
2511 }
2512
2513 return nil
2514}
2515
2516func loopbackOnly(next http.Handler, log zerowrap.Logger) http.Handler {
2517 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2518 host, _, err := net.SplitHostPort(r.RemoteAddr)
2519 if err != nil {
2520 host = r.RemoteAddr
2521 }
2522
2523 ip := net.ParseIP(host)
2524 if ip == nil || !ip.IsLoopback() {
2525 log.Warn().
2526 Str("path", r.URL.Path).
2527 Str("remote_addr", r.RemoteAddr).
2528 Msg("blocked non-loopback access on internal admin route")
2529 w.Header().Set("Content-Type", "application/json")
2530 w.WriteHeader(http.StatusForbidden)
2531 _ = json.NewEncoder(w).Encode(dto.ErrorResponse{Error: "Forbidden"})
2532 return
2533 }
2534
2535 next.ServeHTTP(w, r)
2536 })
2537}
2538
2539// hostRedirectEligible reports whether a known host should be redirected
2540// from plaintext to HTTPS: app hosts declared tls=never stay plain HTTP.
2541func hostRedirectEligible(svc *services, host string) bool {
2542 if svc.appHostIndex == nil {
2543 return true
2544 }
2545 entry, ok := svc.appHostIndex.Lookup(host)
2546 return !ok || entry.TLSMode != domain.AppTLSNever
2547}
2548
2549// hostRequiresTLS reports whether an app host declared tls=always, which
2550// must never be served over plaintext. An unwired index has no always hosts.
2551func hostRequiresTLS(svc *services, host string) bool {
2552 if svc.appHostIndex == nil {
2553 return false
2554 }
2555 entry, ok := svc.appHostIndex.Lookup(host)
2556 return ok && entry.TLSMode == domain.AppTLSAlways
2557}
2558
2559// createHTTPHandlers creates HTTP handlers with middleware.
2560// Returns three handlers: registry, HTTP proxy (with CIDR + onboarding), and HTTPS proxy.
2561func createHTTPHandlers(svc *services, cfg Config, log zerowrap.Logger, accessWriter out.AccessLogWriter) (http.Handler, http.Handler, http.Handler) {
2562 // Parse trusted proxies once for all middleware chains.
2563 // This ensures consistent IP extraction across logging, rate limiting, and auth.
2564 trustedNets := httphelper.ParseTrustedProxies(cfg.API.RateLimit.TrustedProxies)
2565
2566 // Registry handler
2567 registryHandler := registry.NewHandler(svc.registrySvc, log, svc.maxBlobChunkSize, svc.maxBlobSize)
2568 svc.registryHandler = registryHandler
2569 if svc.reloadCoordinator != nil {
2570 svc.reloadCoordinator.SetRegistryLimits(registryHandler)
2571 }
2572 registryWithMiddleware, cidrAllowlistMiddleware, rateLimitMiddleware := buildRegistryHandlerWithMiddleware(
2573 svc,
2574 cfg,
2575 trustedNets,
2576 registryHandler,
2577 log,
2578 )
2579
2580 registryMux := http.NewServeMux()
2581 registerAuthRoutes(registryMux, svc, trustedNets, cidrAllowlistMiddleware, rateLimitMiddleware, cfg, log)
2582 registryMux.Handle("/v2/", wrapRegistryForLocalMode(registryWithMiddleware, cfg, log))
2583 registerAdminRoutes(registryMux, svc, cfg, trustedNets, log)
2584
2585 // Proxy handler. Registry-domain forwarding is gated on auth: with
2586 // auth disabled the registry stays local-only and public ingress can
2587 // never reach it through this proxy.
2588 proxyHandler := proxyadapter.NewHandler(svc.proxySvc, trustedNets, log).
2589 WithRegistryForwarding(cfg.Auth.Enabled)
2590
2591 // HTTP proxy handler chain: HTTPS redirect for non-proxy clients, then CIDR allowlist
2592 proxyAllowedNets, proxyCIDRMiddleware := buildProxyCIDRAllowlistMiddleware(cfg, trustedNets, log)
2593
2594 httpProxyMiddlewares := []func(http.Handler) http.Handler{
2595 middleware.PanicRecovery(log),
2596 middleware.RequestLogger(log, trustedNets),
2597 middleware.SecurityHeaders,
2598 middleware.HTTPSRedirectWithEligibility(proxyAllowedNets, effectiveProxyHTTPPort(cfg), effectiveProxyTLSPort(cfg), cfg.Server.ForceHTTPSRedirect, log, func(host string) bool {
2599 return svc.proxySvc.IsKnownHost(context.Background(), host)
2600 }, func(host string) bool {
2601 return hostRedirectEligible(svc, host)
2602 }, func(host string) bool {
2603 return hostRequiresTLS(svc, host)
2604 }),
2605 }
2606 if proxyCIDRMiddleware != nil {
2607 httpProxyMiddlewares = append(httpProxyMiddlewares, proxyCIDRMiddleware)
2608 }
2609
2610 httpProxyWithMiddleware := otelhttp.NewHandler(
2611 middleware.Chain(httpProxyMiddlewares...)(proxyHandler),
2612 "gordon.proxy",
2613 )
2614
2615 // Build the onboarding handler once if internal CA is available and TLS is enabled.
2616 var obHandler *onboarding.Handler
2617 if svc.caAdapter != nil && effectiveProxyTLSPort(cfg) != 0 {
2618 mobileconfigBytes := pkiadapter.GenerateMobileconfig(
2619 svc.caAdapter.RootCertificateDER(),
2620 svc.caAdapter.RootCommonName(),
2621 )
2622 obHandler = onboarding.NewHandler(
2623 svc.caAdapter.RootCertificate(),
2624 mobileconfigBytes,
2625 svc.caAdapter.RootFingerprint(),
2626 effectiveProxyHTTPPort(cfg),
2627 effectiveProxyTLSPort(cfg),
2628 )
2629 }
2630
2631 // HTTP proxyMux: trusted proxy traffic flows through the normal proxy chain.
2632 // Direct clients get an onboarding gate (when CA is available) placed BEFORE
2633 // HTTPSRedirect so force_https_redirect cannot bypass onboarding.
2634 // ACME HTTP-01 challenge handler is registered before the catch-all "/" so
2635 // it gets first chance regardless of source IP.
2636 proxyMux := http.NewServeMux()
2637
2638 // Register ACME HTTP-01 challenge handler before all other routes so
2639 // Let's Encrypt validation always succeeds, even for onboarding clients.
2640 if svc.publicTLSSvc != nil {
2641 proxyMux.Handle(acmehttp.Prefix, acmehttp.NewHandler(svc.publicTLSSvc))
2642 }
2643
2644 if proxyCIDRMiddleware != nil && proxyAllowedNets == nil {
2645 // Invalid proxy_allowed_ips: deny all traffic (fail-closed).
2646 proxyMux.Handle("/", proxyCIDRMiddleware(httpProxyWithMiddleware))
2647 } else if obHandler != nil {
2648 proxyMux.Handle("/", directHTTPOnboardingGate(obHandler, proxyAllowedNets, httpProxyWithMiddleware, log))
2649 } else {
2650 proxyMux.Handle("/", httpProxyWithMiddleware)
2651 }
2652
2653 // HTTPS proxy handler chain: security headers + proxy + CA onboarding
2654 // Onboarding routes live on the TLS port so Tailnet / direct clients
2655 // can click through the initial cert warning, install the CA, and
2656 // then trust all subsequent connections.
2657 // The middleware chain wraps the entire mux so onboarding routes also
2658 // get PanicRecovery, RequestLogger, and SecurityHeaders.
2659 httpsProxyMiddlewares := []func(http.Handler) http.Handler{
2660 middleware.PanicRecovery(log),
2661 middleware.RequestLogger(log, trustedNets),
2662 middleware.SecurityHeaders,
2663 }
2664
2665 httpsMux := http.NewServeMux()
2666 if obHandler != nil && cfg.Server.GordonDomain != "" {
2667 gordonDomain := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(cfg.Server.GordonDomain)), ".")
2668 onboardingMux := http.NewServeMux()
2669 registerOnboardingRoutes(onboardingMux, obHandler)
2670 // Register onboarding paths host-gated so normal traffic hits
2671 // proxyHandler directly through the catch-all / pattern.
2672 httpsMux.Handle("GET /.well-known/gordon/", gordonDomainOnboardingGate(gordonDomain, onboardingMux, proxyHandler))
2673 httpsMux.Handle("GET /.well-known/gordon/ca", gordonDomainOnboardingGate(gordonDomain, onboardingMux, proxyHandler))
2674 httpsMux.Handle("GET /.well-known/gordon/ca.crt", gordonDomainOnboardingGate(gordonDomain, onboardingMux, proxyHandler))
2675 httpsMux.Handle("GET /.well-known/gordon/ca.mobileconfig", gordonDomainOnboardingGate(gordonDomain, onboardingMux, proxyHandler))
2676 httpsMux.Handle("/", proxyHandler)
2677 } else {
2678 httpsMux.Handle("/", proxyHandler)
2679 }
2680
2681 httpsHandler := otelhttp.NewHandler(middleware.Chain(httpsProxyMiddlewares...)(httpsMux), "gordon.proxy.tls")
2682
2683 // Wrap top-level handlers with access logging outside all gates
2684 // (loopbackOnly, denyAllHandler, CIDR allowlist) so every request —
2685 // including rejected probes — produces exactly one access-log line.
2686 var registryOut, proxyOut, httpsOut http.Handler = registryMux, proxyMux, httpsHandler
2687 if accessWriter != nil {
2688 excludeHC := cfg.Logging.AccessLog.ExcludeHealthChecks
2689 registryOut = middleware.AccessLogger(accessWriter, excludeHC, log, trustedNets)(registryOut)
2690 proxyOut = middleware.AccessLogger(accessWriter, excludeHC, log, trustedNets)(proxyOut)
2691 httpsOut = middleware.AccessLogger(accessWriter, excludeHC, log, trustedNets)(httpsOut)
2692 }
2693 svc.httpsProxyHandler = httpsOut
2694
2695 return registryOut, proxyOut, httpsOut
2696}
2697
2698// registerOnboardingRoutes registers CA onboarding well-known HTTP routes on
2699// the given mux. Both direct-HTTP and Gordon-domain HTTPS onboarding use this.
2700func registerOnboardingRoutes(mux *http.ServeMux, ob *onboarding.Handler) {
2701 mux.HandleFunc("GET /.well-known/gordon/", ob.ServeOnboardingPage)
2702 mux.HandleFunc("GET /.well-known/gordon/ca", ob.ServeOnboardingPage)
2703 mux.HandleFunc("GET /.well-known/gordon/ca.crt", ob.ServeCACert)
2704 mux.HandleFunc("GET /.well-known/gordon/ca.mobileconfig", ob.ServeMobileconfig)
2705}
2706
2707// directHTTPOnboardingGate returns an http.Handler that splits HTTP traffic
2708// by source IP. Trusted proxy IPs flow through to the normal proxy chain.
2709// Direct clients are served the CA onboarding flow on allowed paths and
2710// receive 403 on everything else. This gate runs BEFORE HTTPSRedirect so
2711// force_https_redirect cannot bypass onboarding for direct clients.
2712func directHTTPOnboardingGate(ob *onboarding.Handler, proxyNets []*net.IPNet, proxyChain http.Handler, log zerowrap.Logger) http.Handler {
2713 // Build a small mux for direct-client onboarding paths.
2714 onboardingMux := http.NewServeMux()
2715 registerOnboardingRoutes(onboardingMux, ob)
2716
2717 // Reserve ACME challenge path for future use.
2718 onboardingMux.HandleFunc("/.well-known/acme-challenge/", func(w http.ResponseWriter, _ *http.Request) {
2719 http.Error(w, "not found", http.StatusNotFound)
2720 })
2721
2722 // Catch-all: reject any other direct HTTP request.
2723 // Uses a method-aware split: GET writes a body, HEAD gets an empty 403.
2724 onboardingMux.HandleFunc("/", directHTTPForbidden)
2725
2726 onboardingWithMiddleware := middleware.Chain(
2727 middleware.PanicRecovery(log),
2728 middleware.RequestLogger(log), // Intentionally omit trusted proxy nets so direct onboarding logs use RemoteAddr only.
2729 middleware.SecurityHeaders,
2730 )(onboardingMux)
2731
2732 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2733 remoteIP := httphelper.ExtractRemoteIP(r.RemoteAddr)
2734 if httphelper.IsTrustedOrLocal(remoteIP, proxyNets) {
2735 proxyChain.ServeHTTP(w, r)
2736 return
2737 }
2738 onboardingWithMiddleware.ServeHTTP(w, r)
2739 })
2740}
2741
2742// canonicalHostsEqual compares two hosts after normalising both: stripping
2743// port, trimming spaces, lowercasing, and removing trailing dot.
2744func canonicalHostsEqual(host, expected string) bool {
2745 host = strings.TrimSpace(host)
2746 if h, _, err := net.SplitHostPort(host); err == nil {
2747 host = h
2748 }
2749 host = strings.TrimSuffix(strings.ToLower(host), ".")
2750 expected = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(expected)), ".")
2751 return host == expected
2752}
2753
2754// gordonDomainOnboardingGate returns a handler that serves onboarding routes
2755// only when the request host matches gordonDomain. For mismatched hosts it
2756// delegates to proxyHandler. gordonDomain must already be canonicalised
2757// (trimmed, lowered, trailing dot removed).
2758func gordonDomainOnboardingGate(gordonDomain string, onboardingMux, proxyHandler http.Handler) http.Handler {
2759 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2760 if gordonDomain != "" && canonicalHostsEqual(r.Host, gordonDomain) {
2761 onboardingMux.ServeHTTP(w, r)
2762 return
2763 }
2764 proxyHandler.ServeHTTP(w, r)
2765 })
2766}
2767
2768// directHTTPForbidden responds with 403 for non-onboarding HTTP paths.
2769// HEAD requests get an empty body per HTTP semantics.
2770func directHTTPForbidden(w http.ResponseWriter, r *http.Request) {
2771 w.Header().Set("Content-Type", "text/plain; charset=utf-8")
2772 w.WriteHeader(http.StatusForbidden)
2773 if r.Method != http.MethodHead {
2774 _, _ = w.Write([]byte("Only certificate onboarding is available over HTTP.\n"))
2775 }
2776}
2777
2778func buildRegistryHandlerWithMiddleware(
2779 svc *services,
2780 cfg Config,
2781 trustedNets []*net.IPNet,
2782 registryHandler http.Handler,
2783 log zerowrap.Logger,
2784) (http.Handler, func(http.Handler) http.Handler, func(http.Handler) http.Handler) {
2785 registryMiddlewares := []func(http.Handler) http.Handler{
2786 middleware.PanicRecovery(log),
2787 middleware.RequestLogger(log, trustedNets),
2788 middleware.SecurityHeaders,
2789 }
2790
2791 cidrAllowlistMiddleware := buildRegistryCIDRAllowlistMiddleware(cfg, trustedNets, log)
2792 if cidrAllowlistMiddleware != nil {
2793 registryMiddlewares = append(registryMiddlewares, cidrAllowlistMiddleware)
2794 }
2795
2796 rateLimitMiddleware := buildRegistryRateLimitMiddleware(cfg, log)
2797 registryMiddlewares = append(registryMiddlewares, rateLimitMiddleware)
2798
2799 appendRegistryAuthMiddleware(®istryMiddlewares, svc, cfg, trustedNets, log)
2800
2801 registryWithOtel := otelhttp.NewHandler(
2802 middleware.Chain(registryMiddlewares...)(registryHandler),
2803 "gordon.registry",
2804 )
2805 return registryWithOtel, cidrAllowlistMiddleware, rateLimitMiddleware
2806}
2807
2808// parseCIDRAllowlist parses a list of IPs/CIDRs, logs warnings for invalid entries,
2809// and returns the parsed nets. label is used in log messages (e.g. "registry_allowed_ips").
2810func parseCIDRAllowlist(ips []string, label string, log zerowrap.Logger) ([]*net.IPNet, bool) {
2811 if len(ips) == 0 {
2812 return nil, false
2813 }
2814
2815 allowedNets := httphelper.ParseTrustedProxies(ips)
2816 if len(allowedNets) != len(ips) {
2817 for _, entry := range ips {
2818 if nets := httphelper.ParseTrustedProxies([]string{entry}); len(nets) == 0 {
2819 log.Warn().Str("entry", entry).Msgf("ignoring invalid %s entry", label)
2820 }
2821 }
2822 }
2823
2824 if len(allowedNets) == 0 {
2825 log.Error().
2826 Strs(label, ips).
2827 Msgf("%s is set but no valid entries were parsed; will deny all traffic (fail-closed)", label)
2828 return nil, true // allInvalid
2829 }
2830
2831 return allowedNets, false
2832}
2833
2834// denyAllHandler returns a middleware that rejects every request with 403 Forbidden.
2835func denyAllHandler(label string, trustedNets []*net.IPNet, log zerowrap.Logger) func(http.Handler) http.Handler {
2836 return func(next http.Handler) http.Handler {
2837 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2838 log.Warn().
2839 Str(zerowrap.FieldClientIP, middleware.GetClientIP(r, trustedNets)).
2840 Msgf("access denied due to invalid %s configuration", label)
2841 w.Header().Set("Content-Type", "application/json")
2842 w.WriteHeader(http.StatusForbidden)
2843 _ = json.NewEncoder(w).Encode(dto.ErrorResponse{Error: "Forbidden"})
2844 })
2845 }
2846}
2847
2848func buildRegistryCIDRAllowlistMiddleware(cfg Config, trustedNets []*net.IPNet, log zerowrap.Logger) func(http.Handler) http.Handler {
2849 allowedNets, allInvalid := parseCIDRAllowlist(cfg.Server.RegistryAllowedIPs, "registry_allowed_ips", log)
2850 if allInvalid {
2851 return denyAllHandler("registry_allowed_ips", trustedNets, log)
2852 }
2853 if allowedNets == nil {
2854 return nil
2855 }
2856 return middleware.RegistryCIDRAllowlist(allowedNets, trustedNets, log)
2857}
2858
2859func buildProxyCIDRAllowlistMiddleware(cfg Config, trustedNets []*net.IPNet, log zerowrap.Logger) ([]*net.IPNet, func(http.Handler) http.Handler) {
2860 allowedNets, allInvalid := parseCIDRAllowlist(cfg.Server.ProxyAllowedIPs, "proxy_allowed_ips", log)
2861 if allInvalid {
2862 return nil, denyAllHandler("proxy_allowed_ips", trustedNets, log)
2863 }
2864 if allowedNets == nil {
2865 return nil, nil
2866 }
2867
2868 log.Info().
2869 Strs("proxy_allowed_ips", cfg.Server.ProxyAllowedIPs).
2870 Msg("proxy origin IP allowlist enabled")
2871
2872 return allowedNets, middleware.ProxyCIDRAllowlist(allowedNets, log)
2873}
2874
2875func buildRegistryRateLimitMiddleware(cfg Config, log zerowrap.Logger) func(http.Handler) http.Handler {
2876 if cfg.API.RateLimit.Enabled {
2877 globalLimiter := ratelimit.NewMemoryStore(cfg.API.RateLimit.GlobalRPS, cfg.API.RateLimit.Burst, log)
2878 ipLimiter := ratelimit.NewMemoryStore(cfg.API.RateLimit.PerIPRPS, cfg.API.RateLimit.Burst, log)
2879 return registry.RateLimitMiddleware(
2880 globalLimiter,
2881 ipLimiter,
2882 cfg.API.RateLimit.TrustedProxies,
2883 log,
2884 )
2885 }
2886
2887 return registry.RateLimitMiddleware(nil, nil, nil, log)
2888}
2889
2890func appendRegistryAuthMiddleware(registryMiddlewares *[]func(http.Handler) http.Handler, svc *services, cfg Config, trustedNets []*net.IPNet, log zerowrap.Logger) {
2891 if svc.authSvc != nil {
2892 internalAuth := middleware.InternalRegistryAuth{
2893 Username: svc.internalRegUser,
2894 Password: svc.internalRegPass,
2895 }
2896 *registryMiddlewares = append(*registryMiddlewares, middleware.RegistryAuthV2(svc.authSvc, internalAuth, trustedNets, log))
2897 return
2898 }
2899
2900 if cfg.Auth.Enabled {
2901 log.Error().Msg("authentication service unavailable; registry requests will be denied")
2902 *registryMiddlewares = append(*registryMiddlewares, func(next http.Handler) http.Handler {
2903 return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
2904 w.Header().Set("Content-Type", "application/json")
2905 w.WriteHeader(http.StatusServiceUnavailable)
2906 _ = json.NewEncoder(w).Encode(dto.ErrorResponse{Error: "authentication service unavailable"})
2907 })
2908 })
2909 }
2910}
2911
2912func registerAuthRoutes(
2913 registryMux *http.ServeMux,
2914 svc *services,
2915 trustedNets []*net.IPNet,
2916 cidrAllowlistMiddleware func(http.Handler) http.Handler,
2917 rateLimitMiddleware func(http.Handler) http.Handler,
2918 cfg Config,
2919 log zerowrap.Logger,
2920) {
2921 if svc.authHandler == nil {
2922 return
2923 }
2924
2925 // Auth endpoints always get rate limiting, even if global rate limiting is disabled.
2926 // This prevents brute-force attacks against password/token endpoints.
2927 authRateLimitMiddleware := rateLimitMiddleware
2928 if !cfg.API.RateLimit.Enabled {
2929 authGlobalLimiter := ratelimit.NewMemoryStore(50, 100, log)
2930 authIPLimiter := ratelimit.NewMemoryStore(5, 10, log)
2931 authRateLimitMiddleware = registry.RateLimitMiddleware(authGlobalLimiter, authIPLimiter, cfg.API.RateLimit.TrustedProxies, log)
2932 }
2933
2934 // Auth endpoints are NOT protected by auth - they're where clients authenticate
2935 // but still need rate limiting to prevent brute force attacks.
2936 authMiddlewares := []func(http.Handler) http.Handler{
2937 middleware.PanicRecovery(log),
2938 middleware.RequestLogger(log, trustedNets),
2939 middleware.SecurityHeaders,
2940 }
2941 if cidrAllowlistMiddleware != nil {
2942 authMiddlewares = append(authMiddlewares, cidrAllowlistMiddleware)
2943 }
2944 authMiddlewares = append(authMiddlewares, authRateLimitMiddleware)
2945 authWithMiddleware := otelhttp.NewHandler(
2946 middleware.Chain(authMiddlewares...)(svc.authHandler),
2947 "gordon.auth",
2948 )
2949 registryMux.Handle("/auth/", authWithMiddleware)
2950}
2951
2952func wrapRegistryForLocalMode(registryWithMiddleware http.Handler, cfg Config, log zerowrap.Logger) http.Handler {
2953 if !cfg.Auth.Enabled {
2954 return loopbackOnly(registryWithMiddleware, log)
2955 }
2956 return registryWithMiddleware
2957}
2958
2959func registerAdminRoutes(registryMux *http.ServeMux, svc *services, cfg Config, trustedNets []*net.IPNet, log zerowrap.Logger) {
2960 if svc.adminHandler == nil {
2961 return
2962 }
2963
2964 if !cfg.Auth.Enabled {
2965 log.Warn().Msg("auth disabled: admin API endpoints are not registered")
2966 return
2967 }
2968
2969 adminMiddlewares := []func(http.Handler) http.Handler{
2970 middleware.PanicRecovery(log),
2971 middleware.RequestLogger(log, trustedNets),
2972 middleware.SecurityHeaders,
2973 }
2974
2975 if svc.authSvc != nil {
2976 // Create rate limiters for admin API - uses same config as registry.
2977 var globalLimiter, ipLimiter out.RateLimiter
2978 if cfg.API.RateLimit.Enabled {
2979 globalLimiter = ratelimit.NewMemoryStore(cfg.API.RateLimit.GlobalRPS, cfg.API.RateLimit.Burst, log)
2980 ipLimiter = ratelimit.NewMemoryStore(cfg.API.RateLimit.PerIPRPS, cfg.API.RateLimit.Burst, log)
2981 }
2982 adminMiddlewares = append(adminMiddlewares, admin.AuthMiddleware(svc.authSvc, globalLimiter, ipLimiter, trustedNets, log))
2983 } else {
2984 adminMiddlewares = append(adminMiddlewares, func(next http.Handler) http.Handler {
2985 return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
2986 w.Header().Set("Content-Type", "application/json")
2987 w.WriteHeader(http.StatusServiceUnavailable)
2988 _ = json.NewEncoder(w).Encode(dto.ErrorResponse{Error: "authentication service unavailable"})
2989 })
2990 })
2991 }
2992
2993 adminWithMiddleware := otelhttp.NewHandler(
2994 middleware.Chain(adminMiddlewares...)(svc.adminHandler),
2995 "gordon.admin",
2996 )
2997 registryMux.Handle("/admin/", adminWithMiddleware)
2998}
2999
3000// runServers starts the HTTP servers and waits for shutdown.
3001// Signal handling notes:
3002// - SIGINT/SIGTERM: Triggers graceful shutdown via signal.NotifyContext
3003// - SIGUSR1: Triggers config reload without restart
3004// (SIGUSR2 manual route deploy was removed with the declarative-apps
3005// cutover; app deploys are explicit via `gordon apps deploy`.)
3006// The deferred signal.Stop calls ensure signal handlers are properly
3007// cleaned up before program exit, preventing signal handler leaks.
3008func runServers(ctx context.Context, v *viper.Viper, cfg Config, svc *services, reload reloadTrigger, cleanupHandlers func(), log zerowrap.Logger) error {
3009 // Initialize access log writer. Kept here (not in Run) to keep Run's cyclomatic
3010 // complexity within the project limit of 15.
3011 accessWriterConcrete, err := initAccessLog(cfg, log)
3012 if err != nil {
3013 return err
3014 }
3015 if accessWriterConcrete != nil {
3016 defer accessWriterConcrete.Close()
3017 }
3018 // Convert to interface only when non-nil to avoid the Go nil-interface pitfall
3019 // where a typed nil pointer becomes a non-nil interface value.
3020 var accessWriter out.AccessLogWriter
3021 if accessWriterConcrete != nil {
3022 accessWriter = accessWriterConcrete
3023 }
3024 ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
3025 defer cancel()
3026
3027 // Set up SIGUSR1 for reload.
3028 // Note: signal.Stop must be called (via defer) to release the channel
3029 // and prevent signal handler leaks when the function returns.
3030 reloadChan := make(chan os.Signal, 1)
3031 signal.Notify(reloadChan, syscall.SIGUSR1)
3032 defer signal.Stop(reloadChan)
3033
3034 errChan := make(chan error, 4)
3035
3036 registryHandler, httpProxyHandler, httpsProxyHandler := createHTTPHandlers(svc, cfg, log, accessWriter)
3037 svc.httpProxyHandler = httpProxyHandler
3038 svc.httpsProxyHandler = httpsProxyHandler
3039
3040 registrySrv, registryReady, internalRegistrySrv, internalRegistryReady := startRegistryServers(cfg, registryHandler, errChan, log)
3041
3042 // closeStarted shuts down any servers that were started before an error occurred,
3043 // preventing leaked listeners during partial startup failures.
3044 closeStarted := func(servers ...*http.Server) {
3045 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
3046 defer shutdownCancel()
3047 for _, srv := range servers {
3048 if srv != nil {
3049 if err := srv.Shutdown(shutdownCtx); err != nil {
3050 log.Error().Err(err).Msg("failed to shut down server during startup cleanup")
3051 }
3052 }
3053 }
3054 }
3055
3056 proxySrv, proxyReady, tlsSrv, _, err := startProxyServers(cfg, httpProxyHandler, httpsProxyHandler, svc.pkiSvc, svc.publicTLSSvc, svc.trafficManager, log)
3057 if err != nil {
3058 closeStarted(registrySrv, internalRegistrySrv)
3059 return err
3060 }
3061
3062 // Owner-only local administration socket. Started before readiness is
3063 // announced so the CLI can reach the daemon as soon as it is usable,
3064 // and closed on every exit path. TCP /admin/* registration is unchanged.
3065 localAdmin, err := startLocalAdminServer(svc, errChan, log)
3066 if err != nil {
3067 closeStarted(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3068 shutdownTrafficManagerForStartupCleanup(svc.trafficManager, log)
3069 return err
3070 }
3071
3072 // cleanupAfterLocalAdmin releases the local admin socket and traffic
3073 // manager in addition to the servers that already bound, so no startup
3074 // failure after this point leaks a listening socket or a live manager.
3075 cleanupAfterLocalAdmin := func(servers ...*http.Server) {
3076 cleanupStartupResources(localAdmin, svc.trafficManager, log, servers...)
3077 }
3078
3079 svc.tlsHTTPEntryPoints = tlsMuxHTTPServerNames(cfg)
3080 svc.smartHTTPEntryPoints = smartTCPHTTPServerNames(cfg)
3081
3082 // Wait for both registry listeners and the HTTP proxy before applying traffic.
3083 if err := waitForServerReady(ctx, internalRegistryReady, errChan); err != nil {
3084 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3085 return err
3086 }
3087 if err := waitForCoreProxyReady(ctx, registryReady, proxyReady, errChan); err != nil {
3088 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3089 return err
3090 }
3091 // Apply only the installation graph fail-fast before slower app
3092 // readiness checks. Persisted ACTIVE binds are intentionally excluded
3093 // until boot reconciliation re-inspects them.
3094 if err := applyTrafficRuntimeConfig(ctx, svc.trafficManager, cfg, svc.configSvc, nil); err != nil {
3095 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3096 return err
3097 }
3098
3099 // Reconcile ACTIVE and rebuild its host index before announcing the
3100 // daemon ready. Applying traffic first publishes an empty app graph and
3101 // leaves healthy converged apps returning 404 until a later lifecycle
3102 // operation happens to rebuild it.
3103 reconcileAppsAtBoot(ctx, svc, log)
3104
3105 logEvent := log.Info().
3106 Int("proxy_port", cfg.Server.Port).
3107 Int("registry_port", cfg.Server.RegistryPort)
3108 if cfg.Server.TLSPort != 0 {
3109 logEvent = logEvent.Int("tls_port", cfg.Server.TLSPort)
3110 }
3111 logEvent.Msg("Gordon is running")
3112
3113 startPublicTLSRuntimeWithWarning(ctx, svc.publicTLSRuntime, log)
3114
3115 schedulerCleanup, err := startOptionalSchedulers(ctx, cfg, svc, log, v)
3116 if err != nil {
3117 cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)
3118 return err
3119 }
3120 if schedulerCleanup != nil {
3121 defer schedulerCleanup()
3122 }
3123
3124 waitForShutdown(ctx, errChan, reloadChan, reload, svc.eventBus, log)
3125 cleanupHandlers() // Stop debounce timers before draining containers
3126 // Stop app administration before tearing down traffic/runtime dependencies,
3127 // so no new mutation can begin during shutdown.
3128 localAdmin.Close()
3129 registrySrvs := []*http.Server{registrySrv, internalRegistrySrv}
3130 return gracefulShutdown(registrySrvs, proxySrv, tlsSrv, svc.containerSvc, svc.proxySvc, svc.pkiSvc, svc.publicTLSSvc, svc.trafficManager, svc.appMonitor, svc.appSvcImpl, svc.appState, log)
3131}
3132
3133// reconcileAppsAtBoot reconciles declarative apps intended to run,
3134// rebuilds the proxy host index from reconciled ACTIVE state, then
3135// starts the single periodic recovery monitor. Failures only warn: the
3136// monitor still starts so a partially failed boot self-heals.
3137func reconcileAppsAtBoot(ctx context.Context, svc *services, log zerowrap.Logger) {
3138 if svc.appDeploySvc == nil {
3139 return
3140 }
3141 if err := svc.appDeploySvc.ReconcileBoot(ctx); err != nil {
3142 log.Warn().Err(err).Msg("failed to reconcile apps at boot")
3143 }
3144 if err := rebuildAppHostIndex(ctx, svc); err != nil {
3145 log.Warn().Err(err).Msg("failed to rebuild app host index at boot")
3146 }
3147 // Exactly one monitor per process: a second boot reconciliation must
3148 // not orphan a running loop in the daemon context.
3149 if svc.appMonitor == nil {
3150 svc.appMonitor = newAppMonitor(svc.appDeploySvc, log)
3151 }
3152 svc.appMonitor.Start(ctx)
3153}
3154
3155func startPublicTLSRuntimeWithWarning(ctx context.Context, svc publicTLSRuntime, log zerowrap.Logger) {
3156 if err := startPublicTLSRuntime(ctx, svc, log); err != nil {
3157 log.Warn().Err(err).Msg("initial public ACME reconcile failed, continuing with renewal loop")
3158 }
3159}
3160
3161func waitForCoreProxyReady(ctx context.Context, registryReady <-chan struct{}, proxyReady <-chan struct{}, errChan <-chan error) error {
3162 if err := waitForServerReady(ctx, registryReady, errChan); err != nil {
3163 return err
3164 }
3165 return waitForServerReady(ctx, proxyReady, errChan)
3166}
3167
3168// shutdownStartedServers gracefully shuts down listeners that bound before a
3169// partial startup failure so an error return does not leak them.
3170func shutdownStartedServers(servers []*http.Server, log zerowrap.Logger) {
3171 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
3172 defer shutdownCancel()
3173 for _, srv := range servers {
3174 if srv != nil {
3175 if err := srv.Shutdown(shutdownCtx); err != nil {
3176 log.Error().Err(err).Msg("failed to shut down server during startup cleanup")
3177 }
3178 }
3179 }
3180}
3181
3182// cleanupStartupResources releases every resource a partial startup may have
3183// acquired once the local admin socket is serving: the admin socket first (no
3184// further mutations can begin), then the bound servers, then the traffic
3185// manager that startProxyServers may already have populated.
3186func cleanupStartupResources(localAdmin *localAdminServer, trafficManager *trafficadapter.Manager, log zerowrap.Logger, servers ...*http.Server) {
3187 if localAdmin != nil {
3188 localAdmin.Close()
3189 }
3190 shutdownStartedServers(servers, log)
3191 shutdownTrafficManagerForStartupCleanup(trafficManager, log)
3192}
3193
3194func shutdownTrafficManagerForStartupCleanup(manager *trafficadapter.Manager, log zerowrap.Logger) {
3195 if manager == nil {
3196 return
3197 }
3198 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
3199 defer shutdownCancel()
3200 if err := manager.Shutdown(shutdownCtx); err != nil {
3201 log.Error().Err(err).Msg("failed to shut down traffic manager during startup cleanup")
3202 }
3203}
3204
3205func waitForServerReady(ctx context.Context, ready <-chan struct{}, errChan <-chan error) error {
3206 if ready == nil {
3207 return nil
3208 }
3209 select {
3210 case <-ready:
3211 return nil
3212 case err := <-errChan:
3213 return err
3214 case <-ctx.Done():
3215 return ctx.Err()
3216 }
3217}
3218
3219func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger, v *viper.Viper) (func(), error) {
3220 schedulers := make([]*cronSvc.Scheduler, 0, 3)
3221
3222 backupScheduler, err := startBackupScheduler(ctx, cfg, svc, log)
3223 if err != nil {
3224 return nil, err
3225 }
3226 if backupScheduler != nil {
3227 schedulers = append(schedulers, backupScheduler)
3228 }
3229
3230 volumeBackupScheduler, err := startVolumeBackupScheduler(ctx, cfg, svc, log)
3231 if err != nil {
3232 return nil, err
3233 }
3234 if volumeBackupScheduler != nil {
3235 schedulers = append(schedulers, volumeBackupScheduler)
3236 }
3237
3238 imageScheduler, err := startImagePruneScheduler(ctx, cfg, svc, log, func() int {
3239 return v.GetInt("images.prune.keep_last")
3240 })
3241 if err != nil {
3242 return nil, err
3243 }
3244 if imageScheduler != nil {
3245 schedulers = append(schedulers, imageScheduler)
3246 }
3247
3248 if len(schedulers) == 0 {
3249 return nil, nil
3250 }
3251
3252 return func() {
3253 for i := len(schedulers) - 1; i >= 0; i-- {
3254 schedulers[i].Stop()
3255 }
3256 }, nil
3257}
3258
3259func startBackupScheduler(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger) (*cronSvc.Scheduler, error) {
3260 dbCfg := databaseBackupSettings(cfg)
3261 if !dbCfg.Enabled || svc == nil || svc.backupSvc == nil {
3262 return nil, nil
3263 }
3264
3265 preset, err := resolveBackupSchedule(dbCfg.Schedule)
3266 if err != nil {
3267 return nil, err
3268 }
3269
3270 scheduler := cronSvc.NewScheduler(log)
3271 err = scheduler.Add(
3272 "backup-scheduler",
3273 "Backups",
3274 domain.CronSchedule{Preset: preset},
3275 func(jobCtx context.Context) error {
3276 if err := svc.backupSvc.RunForSchedule(jobCtx, preset); err != nil {
3277 return err
3278 }
3279 log.Info().
3280 Str("schedule", string(preset)).
3281 Msg("scheduled backup run complete")
3282 return nil
3283 },
3284 )
3285 if err != nil {
3286 return nil, log.WrapErr(err, "failed to register backup schedule")
3287 }
3288
3289 scheduler.Start(ctx)
3290 log.Info().
3291 Str("schedule", string(preset)).
3292 Msg("backup scheduler enabled")
3293
3294 return scheduler, nil
3295}
3296
3297func resolveBackupSchedule(raw string) (domain.BackupSchedule, error) {
3298 return resolveSchedulePreset(raw, "backups.databases.schedule", domain.ScheduleDaily)
3299}
3300
3301func startVolumeBackupScheduler(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger) (*cronSvc.Scheduler, error) {
3302 if !cfg.Backups.Volumes.Enabled || svc == nil || svc.volumeBackupSvc == nil {
3303 return nil, nil
3304 }
3305
3306 volumeCfg, err := validateVolumeBackupConfig(cfg)
3307 if err != nil {
3308 return nil, err
3309 }
3310
3311 scheduler := cronSvc.NewScheduler(log)
3312 err = scheduler.Add(
3313 "volume-backup-scheduler",
3314 "Volume Backups",
3315 domain.CronSchedule{Interval: volumeCfg.Interval},
3316 func(jobCtx context.Context) error {
3317 if err := svc.volumeBackupSvc.RunVolumeBackupsForSchedule(jobCtx, ""); err != nil {
3318 return err
3319 }
3320 log.Info().
3321 Dur("interval", volumeCfg.Interval).
3322 Msg("scheduled volume backup run complete")
3323 return nil
3324 },
3325 )
3326 if err != nil {
3327 return nil, log.WrapErr(err, "failed to register volume backup schedule")
3328 }
3329
3330 scheduler.Start(ctx)
3331 log.Info().
3332 Dur("interval", volumeCfg.Interval).
3333 Msg("volume backup scheduler enabled")
3334
3335 return scheduler, nil
3336}
3337
3338func startImagePruneScheduler(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger, keepLastGetter func() int) (*cronSvc.Scheduler, error) {
3339 if !cfg.Images.Prune.Enabled || svc == nil || svc.imageSvc == nil {
3340 return nil, nil
3341 }
3342 // The scheduler runs the same use case and planner as manual
3343 // execution. App presence is normal: apps exist on every real
3344 // server, and prune decides per candidate.
3345 if keepLastGetter == nil {
3346 keepLastGetter = func() int { return cfg.Images.Prune.KeepLast }
3347 }
3348 if keepLastGetter() < 0 {
3349 return nil, fmt.Errorf("images.prune.keep_last must be >= 0")
3350 }
3351
3352 preset, err := resolveImagePruneSchedule(cfg.Images.Prune.Schedule)
3353 if err != nil {
3354 return nil, err
3355 }
3356
3357 scheduler := cronSvc.NewScheduler(log)
3358 err = scheduler.Add(
3359 "image-prune",
3360 "Image prune",
3361 domain.CronSchedule{Preset: preset},
3362 func(jobCtx context.Context) error {
3363 keepLast := keepLastGetter()
3364 if keepLast < 0 {
3365 log.Warn().
3366 Int("configured_keep_last", keepLast).
3367 Int("fallback_keep_last", domain.DefaultImagePruneKeepLast).
3368 Msg("invalid images.prune.keep_last; using default")
3369 keepLast = domain.DefaultImagePruneKeepLast
3370 }
3371
3372 report, err := svc.imageSvc.Prune(jobCtx, domain.ImagePruneOptions{
3373 KeepLast: keepLast,
3374 PruneDangling: true,
3375 PruneRegistry: true,
3376 })
3377 if err != nil {
3378 return err
3379 }
3380
3381 log.Info().
3382 Int("keep_last", keepLast).
3383 Int("eligible", report.Plan.CountByVerdict(domain.PruneVerdictEligible)).
3384 Int("protected", report.Plan.CountByVerdict(domain.PruneVerdictProtected)).
3385 Int("unknown", report.Plan.CountByVerdict(domain.PruneVerdictUnknown)).
3386 Int("deleted", len(report.Plan.Deleted)).
3387 Int("failures", len(report.Plan.Failures)).
3388 Int("inventory_gaps", len(report.Plan.Gaps)).
3389 Int("runtime_deleted", report.Runtime.DeletedCount).
3390 Int("registry_tags_removed", report.Registry.TagsRemoved).
3391 Int("registry_blobs_removed", report.Registry.BlobsRemoved).
3392 Msg("scheduled image prune complete")
3393 return nil
3394 },
3395 )
3396 if err != nil {
3397 return nil, log.WrapErr(err, "failed to register image prune schedule")
3398 }
3399
3400 scheduler.Start(ctx)
3401 log.Info().
3402 Str("schedule", string(preset)).
3403 Int("keep_last", keepLastGetter()).
3404 Msg("image prune scheduler enabled")
3405
3406 return scheduler, nil
3407}
3408
3409func resolveImagePruneSchedule(raw string) (domain.BackupSchedule, error) {
3410 return resolveSchedulePreset(raw, "images.prune.schedule", domain.ScheduleDaily)
3411}
3412
3413func resolveSchedulePreset(raw, name string, defaultVal domain.BackupSchedule) (domain.BackupSchedule, error) {
3414 schedule := domain.BackupSchedule(strings.ToLower(strings.TrimSpace(raw)))
3415 if schedule == "" {
3416 schedule = defaultVal
3417 }
3418
3419 switch schedule {
3420 case domain.ScheduleHourly, domain.ScheduleDaily, domain.ScheduleWeekly, domain.ScheduleMonthly:
3421 return schedule, nil
3422 default:
3423 return "", fmt.Errorf("%s must be one of: hourly, daily, weekly, monthly", name)
3424 }
3425}
3426
3427// waitForShutdown blocks on the event loop, handling server errors and
3428// Unix signals (reload, deploy, shutdown) until the context is cancelled.
3429func waitForShutdown(ctx context.Context, errChan <-chan error, reloadChan <-chan os.Signal, reload reloadTrigger, eventBus out.EventBus, log zerowrap.Logger) {
3430 for {
3431 select {
3432 case err := <-errChan:
3433 log.Error().Err(err).Msg("server error")
3434 return
3435 case <-reloadChan:
3436 log.Info().Msg("reload signal received (SIGUSR1)")
3437 _ = reload.Trigger(ctx)
3438 case <-ctx.Done():
3439 log.Info().Msg("shutdown signal received")
3440 return
3441 }
3442 }
3443}
3444
3445// appAdministration is the daemon-owned application lifecycle torn down
3446// during graceful shutdown: it refuses new background deploys, cancels
3447// in-flight executions on the daemon context, and joins them.
3448type appAdministration interface {
3449 Shutdown(context.Context) error
3450}
3451
3452// quiesceAppAdministration cancels and joins daemon-owned app work on the
3453// bounded shutdown context. A non-nil error means the bound expired while an
3454// execution was still unwinding: the caller must stop tearing down the state
3455// and runtime that execution still uses.
3456func quiesceAppAdministration(ctx context.Context, appAdmin appAdministration, log zerowrap.Logger) error {
3457 if appAdmin == nil {
3458 return nil
3459 }
3460 if err := appAdmin.Shutdown(ctx); err != nil {
3461 log.Error().Err(err).Msg("app administration did not quiesce before the shutdown deadline")
3462 return err
3463 }
3464 return nil
3465}
3466
3467// gracefulShutdown stops HTTP servers with a 30s timeout, then shuts down
3468// the container service and cleans up runtime files. It returns an error when
3469// app administration could not quiesce: in that case the remaining teardown
3470// is skipped and the error is propagated to the process entry point, which
3471// exits non-zero instead of closing state or runtime under an in-flight
3472// ExecuteDeploy.
3473func gracefulShutdown(registrySrvs []*http.Server, proxySrv, tlsSrv *http.Server, containerSvc *container.Service, proxySvc *proxy.Service, pkiSvc *pkiusecase.Service, publicTLS in.PublicTLSService, trafficManager *trafficadapter.Manager, monitor *appMonitor, appAdmin appAdministration, appState out.AppState, log zerowrap.Logger) error {
3474 log.Info().Msg("shutting down Gordon...")
3475
3476 // Phase 0: stop periodic app recovery first and wait for any in-flight
3477 // bounded pass, so no reconcile can race runtime, traffic, or state
3478 // teardown (and no goroutine is left behind).
3479 monitor.Stop()
3480
3481 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
3482 defer shutdownCancel()
3483
3484 // Phase 0.5: quiesce daemon-owned app administration. New background
3485 // deploys are refused and in-flight executions are cancelled on the
3486 // daemon context and joined here — before traffic, runtime, or the app
3487 // state store is torn down, so no execution can write to closed state.
3488 // A timeout is fail-closed: the unfinished execution still owns the
3489 // state and runtime it uses, so teardown stops here and the error makes
3490 // the process exit non-zero rather than close resources underneath it.
3491 if err := quiesceAppAdministration(shutdownCtx, appAdmin, log); err != nil {
3492 return fmt.Errorf("app administration quiescence: %w", err)
3493 }
3494
3495 // Phase 1: Stop ingress frontends (TLS, then proxy) — no new traffic accepted
3496 shutdownHTTPServers(shutdownCtx, log, tlsSrv, proxySrv)
3497
3498 if trafficManager != nil {
3499 if err := trafficManager.Shutdown(shutdownCtx); err != nil {
3500 log.Warn().Err(err).Msg("traffic manager shutdown error")
3501 }
3502 }
3503
3504 // Stop PKI maintenance goroutines
3505 if pkiSvc != nil {
3506 pkiSvc.Stop()
3507 }
3508
3509 // Stop public ACME TLS renewal loop
3510 if publicTLS != nil {
3511 if err := publicTLS.Stop(shutdownCtx); err != nil {
3512 log.Warn().Err(err).Msg("public TLS stop error")
3513 }
3514 }
3515
3516 // Phase 2: Drain in-flight registry push sessions before stopping the backend
3517 if proxySvc != nil {
3518 log.Info().Msg("draining in-flight registry requests...")
3519 if drained := proxySvc.DrainRegistryInFlight(25 * time.Second); !drained {
3520 log.Warn().Int64("in_flight", proxySvc.RegistryInFlight()).Msg("registry drain timed out; some in-flight pushes may be interrupted")
3521 }
3522 }
3523
3524 // Phase 3: Stop the registry backends.
3525 shutdownHTTPServers(shutdownCtx, log, registrySrvs...)
3526
3527 if err := containerSvc.Shutdown(shutdownCtx); err != nil {
3528 log.Warn().Err(err).Msg("error during container shutdown")
3529 }
3530 closeAppState(appState, log)
3531
3532 cleanupInternalCredentials()
3533 log.Info().Msg("Gordon stopped")
3534 return nil
3535}
3536
3537// shutdownHTTPServers gracefully stops each non-nil server on the shutdown
3538// context, logging per-server failures so one listener cannot mask another.
3539func shutdownHTTPServers(ctx context.Context, log zerowrap.Logger, servers ...*http.Server) {
3540 for _, srv := range servers {
3541 if srv == nil {
3542 continue
3543 }
3544 if err := srv.Shutdown(ctx); err != nil {
3545 log.Warn().Err(err).Str("addr", srv.Addr).Msg("server shutdown error")
3546 }
3547 }
3548}
3549
3550func closeAppState(state out.AppState, log zerowrap.Logger) {
3551 if state == nil {
3552 return
3553 }
3554 if err := state.Close(); err != nil {
3555 log.Warn().Err(err).Msg("app state close error")
3556 }
3557}
3558
3559// startProxyServers sets up the HTTP proxy server and, when tls_port != 0,
3560// an HTTPS proxy server with on-demand TLS certificates from the internal CA.
3561// certificateSelector implements a multi-source TLS certificate lookup.
3562// Priority: static certs → public ACME TLS → local PKI (internal CA).
3563type certificateSelector struct {
3564 staticCerts []staticTLSCertificate
3565 publicTLS in.PublicTLSService
3566 localPKI *pkiusecase.Service
3567}
3568
3569type staticTLSCertificate struct {
3570 cert tls.Certificate
3571 leaf *x509.Certificate
3572}
3573
3574// GetCertificate selects a TLS certificate based on the ClientHello SNI.
3575//
3576// Priority:
3577// 1. Static certs — exact SNI match (leaf VerifyHostname)
3578// 2. Public ACME TLS — if the host requires ACME coverage
3579// 3. Local PKI (internal CA) — fallback for all other hosts
3580// 4. nil, nil — if no source can serve the host
3581func (s *certificateSelector) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
3582 // 1. Static certs — exact match via leaf VerifyHostname.
3583 if cert := matchingPreparedStaticCert(s.staticCerts, hello.ServerName); cert != nil {
3584 return cert, nil
3585 }
3586
3587 // 2. Public ACME TLS.
3588 if s.publicTLS != nil {
3589 cert, err := s.publicTLS.GetCertificateForHost(hello.ServerName)
3590 if err == nil && cert != nil {
3591 return cert, nil
3592 }
3593 // nil, nil means this host is not an ACME-required route. Errors mean
3594 // public ACME cannot currently serve this host. In both cases, fall
3595 // through to local PKI instead of aborting the TLS handshake.
3596 }
3597
3598 // 3. Local PKI (internal CA).
3599 if s.localPKI != nil {
3600 return s.localPKI.GetCertificate(hello)
3601 }
3602
3603 return nil, nil
3604}
3605
3606func prepareStaticTLSCertificates(certs []tls.Certificate) []staticTLSCertificate {
3607 prepared := make([]staticTLSCertificate, 0, len(certs))
3608 for _, cert := range certs {
3609 if cert.Leaf == nil && len(cert.Certificate) > 0 {
3610 leaf, err := x509.ParseCertificate(cert.Certificate[0])
3611 if err == nil {
3612 cert.Leaf = leaf
3613 }
3614 }
3615 prepared = append(prepared, staticTLSCertificate{cert: cert, leaf: cert.Leaf})
3616 }
3617 return prepared
3618}
3619
3620func matchingPreparedStaticCert(certs []staticTLSCertificate, serverName string) *tls.Certificate {
3621 if serverName == "" {
3622 if len(certs) == 0 {
3623 return nil
3624 }
3625 return &certs[0].cert
3626 }
3627 for i := range certs {
3628 if certs[i].leaf == nil {
3629 continue
3630 }
3631 if err := certs[i].leaf.VerifyHostname(serverName); err == nil {
3632 return &certs[i].cert
3633 }
3634 }
3635 return nil
3636}
3637
3638// matchingStaticCert returns a pointer to the first static certificate whose
3639// leaf verifies the given serverName. Returns nil if no match is found.
3640func matchingStaticCert(certs []tls.Certificate, serverName string) *tls.Certificate {
3641 return matchingPreparedStaticCert(prepareStaticTLSCertificates(certs), serverName)
3642}
3643
3644func startProxyServers(cfg Config, httpHandler, httpsHandler http.Handler, pkiSvc *pkiusecase.Service, publicTLS in.PublicTLSService, trafficManager *trafficadapter.Manager, log zerowrap.Logger) (*http.Server, <-chan struct{}, *http.Server, <-chan struct{}, error) {
3645 var httpSrv *http.Server
3646 var httpReady <-chan struct{}
3647
3648 needsTLS := hasTLSCapableEntrypoint(cfg)
3649 if !hasSmartTCPEntrypoint(cfg) && !needsTLS {
3650 return httpSrv, httpReady, nil, nil, nil
3651 }
3652 cleanupHTTP := func(err error) (*http.Server, <-chan struct{}, *http.Server, <-chan struct{}, error) {
3653 return httpSrv, httpReady, nil, nil, err
3654 }
3655
3656 var tlsConfig *tls.Config
3657 if needsTLS {
3658 var err error
3659 tlsConfig, err = proxyTLSConfig(cfg, pkiSvc, publicTLS, log)
3660 if err != nil {
3661 return cleanupHTTP(err)
3662 }
3663 }
3664 if trafficManager == nil {
3665 return cleanupHTTP(fmt.Errorf("traffic manager is required when traffic entrypoints are enabled"))
3666 }
3667 registerTLSMuxHTTPServers(trafficManager, cfg, httpsHandler, tlsConfig, nil)
3668 registerSmartTCPHTTPServers(trafficManager, cfg, httpHandler, httpsHandler, tlsConfig, nil)
3669
3670 return httpSrv, httpReady, nil, nil, nil
3671}
3672
3673func hasSmartTCPEntrypoint(cfg Config) bool {
3674 for _, entryPoint := range cfg.EntryPoints {
3675 if entryPoint.Protocol == domain.EntryPointProtocolSmartTCP {
3676 return true
3677 }
3678 }
3679 return false
3680}
3681
3682func hasTLSCapableEntrypoint(cfg Config) bool {
3683 for _, entryPoint := range cfg.EntryPoints {
3684 switch entryPoint.Protocol {
3685 case domain.EntryPointProtocolSmartTCP, domain.EntryPointProtocolTLSMux:
3686 return true
3687 }
3688 }
3689 return false
3690}
3691
3692func effectivePublicTLSPort(cfg Config) int {
3693 return effectiveEntrypointPort(cfg, tlsCapableEntryPoint)
3694}
3695
3696func effectiveProxyTLSPort(cfg Config) int {
3697 return effectivePublicTLSPort(cfg)
3698}
3699
3700func effectiveProxyHTTPPort(cfg Config) int {
3701 return effectiveEntrypointPort(cfg, func(protocol domain.EntryPointProtocol) bool {
3702 return protocol == domain.EntryPointProtocolSmartTCP
3703 })
3704}
3705
3706func effectiveEntrypointPort(cfg Config, match func(domain.EntryPointProtocol) bool) int {
3707 if entryPoint, ok := cfg.EntryPoints[traffic.DefaultEdgeEntryPointName]; ok && match(entryPoint.Protocol) {
3708 if port := portFromAddress(entryPoint.Address); port > 0 {
3709 return port
3710 }
3711 }
3712
3713 var candidatePort int
3714 candidates := 0
3715 for name, entryPoint := range cfg.EntryPoints {
3716 if name == traffic.DefaultEdgeEntryPointName || !match(entryPoint.Protocol) {
3717 continue
3718 }
3719 port := portFromAddress(entryPoint.Address)
3720 if port == 0 {
3721 continue
3722 }
3723 candidatePort = port
3724 candidates++
3725 }
3726 if candidates == 1 {
3727 return candidatePort
3728 }
3729 return 0
3730}
3731
3732func tlsCapableEntryPoint(protocol domain.EntryPointProtocol) bool {
3733 switch protocol {
3734 case domain.EntryPointProtocolSmartTCP, domain.EntryPointProtocolTLSMux:
3735 return true
3736 default:
3737 return false
3738 }
3739}
3740
3741func effectiveHTTP01Port(cfg Config) int {
3742 if hasSmartTCPHTTP01Entrypoint(cfg) {
3743 return 80
3744 }
3745 return 0
3746}
3747
3748func validatePublicTLSReadiness(cfg Config) error {
3749 mode, err := domain.ParseACMEChallengeMode(cfg.TLS.ACME.Challenge)
3750 if err != nil {
3751 return err
3752 }
3753 switch mode {
3754 case domain.ACMEChallengeCloudflareDNS01, domain.ACMEChallengeAuto:
3755 return nil
3756 case domain.ACMEChallengeHTTP01:
3757 return validateHTTP01ChallengeReadiness(cfg)
3758 default:
3759 return fmt.Errorf("%w: %q", domain.ErrACMEChallengeInvalid, mode)
3760 }
3761}
3762
3763func validateEffectivePublicTLSReadiness(cfg Config, effective publictls.EffectiveChallenge) error {
3764 if effective.Mode != domain.ACMEChallengeHTTP01 {
3765 return nil
3766 }
3767 return validateHTTP01ChallengeReadiness(cfg)
3768}
3769
3770func validateHTTP01ChallengeReadiness(cfg Config) error {
3771 if hasBoundHTTP01ChallengeListener(cfg) {
3772 return nil
3773 }
3774 return fmt.Errorf("%w: http-01 requires an actually bound HTTP-01 challenge listener on external :80", domain.ErrACMEChallengeInvalid)
3775}
3776
3777func hasBoundHTTP01ChallengeListener(cfg Config) bool {
3778 return hasSmartTCPHTTP01Entrypoint(cfg)
3779}
3780
3781func hasSmartTCPHTTP01Entrypoint(cfg Config) bool {
3782 for _, entryPoint := range cfg.EntryPoints {
3783 if entryPoint.Protocol == domain.EntryPointProtocolSmartTCP && portFromAddress(entryPoint.Address) == 80 {
3784 return true
3785 }
3786 }
3787 return false
3788}
3789
3790func portFromAddress(address string) int {
3791 _, portText, err := net.SplitHostPort(address)
3792 if err != nil {
3793 return 0
3794 }
3795 port, err := strconv.Atoi(portText)
3796 if err != nil {
3797 return 0
3798 }
3799 return port
3800}
3801
3802func proxyTLSConfig(cfg Config, pkiSvc *pkiusecase.Service, publicTLS in.PublicTLSService, log zerowrap.Logger) (*tls.Config, error) {
3803 var staticCerts []tls.Certificate
3804 if cfg.Server.TLSCertFile != "" {
3805 staticCert, err := tls.LoadX509KeyPair(cfg.Server.TLSCertFile, cfg.Server.TLSKeyFile)
3806 if err != nil {
3807 return nil, fmt.Errorf("load TLS keypair: %w", err)
3808 }
3809 staticCerts = []tls.Certificate{staticCert}
3810 log.Info().
3811 Str("cert", cfg.Server.TLSCertFile).
3812 Str("key", cfg.Server.TLSKeyFile).
3813 Msg("loaded static TLS certificate (public ACME and internal CA handle remaining domains)")
3814 }
3815 selector := &certificateSelector{
3816 staticCerts: prepareStaticTLSCertificates(staticCerts),
3817 publicTLS: publicTLS,
3818 localPKI: pkiSvc,
3819 }
3820 return &tls.Config{
3821 MinVersion: tls.VersionTLS12,
3822 GetCertificate: selector.GetCertificate,
3823 NextProtos: []string{"h2", "http/1.1"},
3824 }, nil
3825}
3826
3827func registerSmartTCPHTTPServers(manager *trafficadapter.Manager, cfg Config, httpHandler, httpsHandler http.Handler, tlsConfig *tls.Config, previous map[string]struct{}) map[string]struct{} {
3828 if manager == nil {
3829 return previous
3830 }
3831 next := smartTCPHTTPServerNames(cfg)
3832 for name := range previous {
3833 if _, ok := next[name]; !ok {
3834 manager.SetSmartTCPHTTPServer(name, nil, nil)
3835 manager.SetSmartTCPTLSServer(name, nil, nil)
3836 }
3837 }
3838 var httpProtos http.Protocols
3839 httpProtos.SetHTTP1(true)
3840 httpProtos.SetUnencryptedHTTP2(true)
3841 for name := range next {
3842 if httpHandler != nil {
3843 manager.SetSmartTCPHTTPServer(name, httpHandler, &httpProtos)
3844 }
3845 if httpsHandler != nil && tlsConfig != nil {
3846 manager.SetSmartTCPTLSServer(name, httpsHandler, tlsConfig)
3847 }
3848 }
3849 return next
3850}
3851
3852func smartTCPHTTPServerNames(cfg Config) map[string]struct{} {
3853 names := map[string]struct{}{}
3854 for name, entryPoint := range cfg.EntryPoints {
3855 if entryPoint.Protocol == domain.EntryPointProtocolSmartTCP {
3856 names[name] = struct{}{}
3857 }
3858 }
3859 return names
3860}
3861
3862func registerTLSMuxHTTPServers(manager *trafficadapter.Manager, cfg Config, httpsHandler http.Handler, tlsConfig *tls.Config, previous map[string]struct{}) map[string]struct{} {
3863 if manager == nil {
3864 return previous
3865 }
3866 next := tlsMuxHTTPServerNames(cfg)
3867 for name := range previous {
3868 if _, ok := next[name]; !ok {
3869 manager.SetTLSHTTPServer(name, nil, nil)
3870 }
3871 }
3872 if httpsHandler == nil || tlsConfig == nil {
3873 return next
3874 }
3875 for name := range next {
3876 manager.SetTLSHTTPServer(name, httpsHandler, tlsConfig)
3877 }
3878 return next
3879}
3880
3881func tlsMuxHTTPServerNames(cfg Config) map[string]struct{} {
3882 names := map[string]struct{}{}
3883 for name, entryPoint := range cfg.EntryPoints {
3884 domainEntryPoint := domain.EntryPoint{Name: name, Protocol: entryPoint.Protocol}
3885 if trafficManagerOwnsEntryPoint(domainEntryPoint) && domainEntryPoint.Protocol == domain.EntryPointProtocolTLSMux {
3886 names[name] = struct{}{}
3887 }
3888 }
3889 return names
3890}
3891
3892func startRegistryServers(cfg Config, handler http.Handler, errChan chan<- error, log zerowrap.Logger) (*http.Server, <-chan struct{}, *http.Server, <-chan struct{}) {
3893 port := strconv.Itoa(cfg.Server.RegistryPort)
3894 externalAddr := net.JoinHostPort(cfg.Server.RegistryListenAddr, port)
3895 externalSrv, externalReady := startServer(externalAddr, handler, "registry", nil, errChan, log)
3896 internalReady := closedReadyChannel()
3897 if !needsInternalRegistryListener(cfg.Server.RegistryListenAddr) {
3898 return externalSrv, externalReady, nil, internalReady
3899 }
3900 internalAddr := net.JoinHostPort("127.0.0.1", port)
3901 internalSrv, internalReady := startServer(internalAddr, handler, "internal registry", nil, errChan, log)
3902 return externalSrv, externalReady, internalSrv, internalReady
3903}
3904
3905func closedReadyChannel() <-chan struct{} {
3906 ready := make(chan struct{})
3907 close(ready)
3908 return ready
3909}
3910
3911func needsInternalRegistryListener(listenAddr string) bool {
3912 addr := net.ParseIP(strings.TrimSpace(listenAddr))
3913 if addr == nil {
3914 return listenAddr != ""
3915 }
3916 return !addr.IsLoopback() && !addr.IsUnspecified()
3917}
3918
3919// startServer starts an HTTP server, returning the server instance and a channel
3920// that closes once the listening socket is bound. This lets callers wait for the
3921// port to be ready before taking actions that depend on it (e.g. auto-start
3922// pulling from the local registry). The returned *http.Server can be used for
3923// graceful shutdown.
3924func startServer(addr string, handler http.Handler, name string, protocols *http.Protocols, errChan chan<- error, log zerowrap.Logger) (*http.Server, <-chan struct{}) {
3925 ready := make(chan struct{})
3926
3927 server := &http.Server{
3928 Addr: addr,
3929 Handler: handler,
3930 Protocols: protocols,
3931 ReadHeaderTimeout: 10 * time.Second,
3932 ReadTimeout: 5 * time.Minute,
3933 WriteTimeout: 5 * time.Minute,
3934 IdleTimeout: 120 * time.Second,
3935 MaxHeaderBytes: 1 << 20,
3936 }
3937
3938 go func() {
3939 log.Info().Str("address", addr).Msgf("%s server starting", name)
3940
3941 ln, err := net.Listen("tcp", addr)
3942 if err != nil {
3943 errChan <- fmt.Errorf("%s server error: %w", name, err)
3944 return
3945 }
3946 close(ready) // signal: port is bound and accepting connections
3947
3948 if err := server.Serve(ln); err != nil && err != http.ErrServerClosed {
3949 errChan <- fmt.Errorf("%s server error: %w", name, err)
3950 }
3951 }()
3952
3953 return server, ready
3954}
3955
3956// SendReloadSignal sends SIGUSR1 to the running Gordon process.
3957func SendReloadSignal() error {
3958 process, _, err := findRunningProcess()
3959 if err != nil {
3960 return err
3961 }
3962
3963 if err := process.Signal(syscall.SIGUSR1); err != nil {
3964 return fmt.Errorf("failed to send reload signal: %w", err)
3965 }
3966
3967 return nil
3968}
3969
3970// createPidFile creates a PID file for the Gordon process.
3971// SECURITY: Prefers secure locations (XDG_RUNTIME_DIR, ~/.gordon/run) over /tmp
3972// to prevent symlink attacks and unauthorized access.
3973func createPidFile(log zerowrap.Logger) string {
3974 pid := os.Getpid()
3975
3976 // SECURITY: Prioritize secure locations over /tmp
3977 var locations []string
3978
3979 // Try secure runtime directory first
3980 if runtimeDir, err := getSecureRuntimeDir(); err == nil {
3981 locations = append(locations, filepath.Join(runtimeDir, "gordon.pid"))
3982 }
3983
3984 // Fall back to home directory
3985 if homeDir, err := os.UserHomeDir(); err == nil {
3986 locations = append(locations, filepath.Join(homeDir, ".gordon", "gordon.pid"))
3987 }
3988
3989 // Last resort: /tmp (least secure due to world-writable)
3990 locations = append(locations, filepath.Join(os.TempDir(), "gordon.pid"))
3991
3992 for _, location := range locations {
3993 // Ensure parent directory exists with secure permissions
3994 if err := os.MkdirAll(filepath.Dir(location), 0700); err != nil {
3995 continue
3996 }
3997 if err := os.WriteFile(location, fmt.Appendf(nil, "%d", pid), 0600); err == nil {
3998 log.Debug().Str("pid_file", location).Int("pid", pid).Msg("created PID file")
3999 return location
4000 }
4001 }
4002
4003 log.Warn().Int("pid", pid).Msg("failed to create PID file in any location")
4004 return ""
4005}
4006
4007// removePidFile removes the PID file.
4008func removePidFile(pidFile string, log zerowrap.Logger) {
4009 if err := os.Remove(pidFile); err != nil {
4010 log.Warn().Err(err).Str("pid_file", pidFile).Msg("failed to remove PID file")
4011 } else {
4012 log.Debug().Str("pid_file", pidFile).Msg("removed PID file")
4013 }
4014}
4015
4016func pidFileLocations() []string {
4017 var locations []string
4018 seen := make(map[string]struct{})
4019 add := func(path string) {
4020 if path == "" {
4021 return
4022 }
4023 if _, ok := seen[path]; ok {
4024 return
4025 }
4026 seen[path] = struct{}{}
4027 locations = append(locations, path)
4028 }
4029
4030 // Check secure runtime directory first
4031 if runtimeDir, err := getSecureRuntimeDir(); err == nil {
4032 add(filepath.Join(runtimeDir, "gordon.pid"))
4033 }
4034
4035 // Also check canonical /run/user/<uid> runtime path. This handles cases where
4036 // Gordon started under systemd user services with runtime dir available, but
4037 // CLI invocations (e.g. non-interactive SSH) don't have XDG_RUNTIME_DIR set.
4038 runtimeByUID := filepath.Join("/run/user", strconv.Itoa(os.Getuid()), "gordon", "gordon.pid")
4039 add(runtimeByUID)
4040
4041 // Check explicit XDG_RUNTIME_DIR if present in this process env.
4042 if runtimeDir := os.Getenv("XDG_RUNTIME_DIR"); runtimeDir != "" {
4043 add(filepath.Join(runtimeDir, "gordon.pid"))
4044 }
4045
4046 // Check home directory
4047 if homeDir, err := os.UserHomeDir(); err == nil {
4048 add(filepath.Join(homeDir, ".gordon", "gordon.pid"))
4049 // Legacy location for backward compatibility
4050 add(filepath.Join(homeDir, ".gordon.pid"))
4051 }
4052
4053 // Legacy /tmp locations for backward compatibility
4054 add(filepath.Join(os.TempDir(), "gordon.pid"))
4055 add("/tmp/gordon.pid")
4056
4057 return locations
4058}
4059
4060// findRunningPidFile returns the first PID file whose PID belongs to a live process.
4061// Stale/invalid PID files are ignored and removed when possible.
4062func findRunningPidFile() (string, int, error) {
4063 return findRunningPidFileInLocations(pidFileLocations())
4064}
4065
4066func findRunningPidFileInLocations(locations []string) (string, int, error) {
4067 foundAny := false
4068
4069 for _, location := range locations {
4070 pidBytes, err := os.ReadFile(location)
4071 if err != nil {
4072 continue
4073 }
4074
4075 foundAny = true
4076
4077 var pid int
4078 if _, err := fmt.Sscanf(string(pidBytes), "%d", &pid); err != nil || pid <= 0 {
4079 _ = os.Remove(location)
4080 continue
4081 }
4082
4083 if isProcessAlive(pid) {
4084 return location, pid, nil
4085 }
4086
4087 _ = os.Remove(location)
4088 }
4089
4090 if foundAny {
4091 return "", 0, fmt.Errorf("found stale gordon PID file(s), is Gordon running?")
4092 }
4093
4094 return "", 0, fmt.Errorf("gordon PID file not found, is Gordon running?")
4095}
4096
4097func findRunningProcess() (*os.Process, int, error) {
4098 _, pid, err := findRunningPidFile()
4099 if err != nil {
4100 return nil, 0, err
4101 }
4102
4103 process, err := os.FindProcess(pid)
4104 if err != nil {
4105 return nil, 0, fmt.Errorf("failed to find process: %w", err)
4106 }
4107
4108 return process, pid, nil
4109}
4110
4111func isProcessAlive(pid int) bool {
4112 process, err := os.FindProcess(pid)
4113 if err != nil {
4114 return false
4115 }
4116
4117 err = process.Signal(syscall.Signal(0))
4118 if err == nil {
4119 return true
4120 }
4121
4122 return errors.Is(err, syscall.EPERM)
4123}
4124
4125// loadConfig loads configuration from file and sets defaults.
4126func loadConfig(v *viper.Viper, configPath string) error {
4127 v.SetDefault("server.registry_port", 5000)
4128 v.SetDefault("server.legacy_registry_domains", []string{})
4129 v.SetDefault("server.tls_cert_file", "")
4130 v.SetDefault("server.tls_key_file", "")
4131 v.SetDefault("tls.acme.enabled", false)
4132 v.SetDefault("tls.acme.email", "")
4133 v.SetDefault("tls.acme.challenge", "auto")
4134 v.SetDefault("tls.acme.obtain_batch_size", 1)
4135 v.SetDefault("dns.resolvers", publictls.DefaultDNSResolvers)
4136 v.SetDefault("dns.propagation_timeout", "5m")
4137 v.SetDefault("dns.polling_interval", "5s")
4138 v.SetDefault("server.force_https_redirect", false)
4139 v.SetDefault("server.data_dir", DefaultDataDir())
4140 v.SetDefault("server.runtime", "auto")
4141 v.SetDefault("logging.level", "info")
4142 v.SetDefault("logging.format", "console")
4143 v.SetDefault("logging.file.enabled", false)
4144 v.SetDefault("logging.file.max_size", 100)
4145 v.SetDefault("logging.file.max_backups", 3)
4146 v.SetDefault("logging.file.max_age", 28)
4147 v.SetDefault("logging.container_logs.enabled", true)
4148 v.SetDefault("logging.container_logs.dir", "")
4149 v.SetDefault("logging.container_logs.max_size", 100)
4150 v.SetDefault("logging.container_logs.max_backups", 3)
4151 v.SetDefault("logging.container_logs.max_age", 28)
4152 v.SetDefault("logging.access_log.enabled", false)
4153 v.SetDefault("logging.access_log.format", "json")
4154 v.SetDefault("logging.access_log.output", "stdout")
4155 v.SetDefault("logging.access_log.file_path", "")
4156 v.SetDefault("logging.access_log.max_size", 100)
4157 v.SetDefault("logging.access_log.max_backups", 3)
4158 v.SetDefault("logging.access_log.max_age", 28)
4159 v.SetDefault("logging.access_log.exclude_health_checks", true)
4160 v.SetDefault("logging.access_log.syslog_identifier", "gordon-access")
4161 v.SetDefault("env.dir", "") // defaults to {data_dir}/env when empty
4162 v.SetDefault("auth.enabled", true)
4163 // Note: auth.type defaults to "token" (the only supported mode)
4164 v.SetDefault("auth.secrets_backend", "")
4165 v.SetDefault("auth.token_expiry", "720h")
4166 v.SetDefault("api.rate_limit.enabled", true)
4167 v.SetDefault("api.rate_limit.global_rps", 500)
4168 v.SetDefault("api.rate_limit.per_ip_rps", 50)
4169 v.SetDefault("api.rate_limit.burst", 100)
4170 v.SetDefault("auto_route.enabled", false)
4171 v.SetDefault("network_isolation.enabled", true)
4172 v.SetDefault("network_isolation.network_prefix", "gordon")
4173 v.SetDefault("network_isolation.internal", false)
4174 v.SetDefault("volumes.auto_create", true)
4175 v.SetDefault("volumes.prefix", "gordon")
4176 v.SetDefault("volumes.preserve", true)
4177 v.SetDefault("backups.databases.enabled", false)
4178 v.SetDefault("backups.databases.schedule", string(domain.ScheduleDaily))
4179 v.SetDefault("backups.databases.storage_dir", "")
4180 v.SetDefault("backups.databases.retention.hourly", 0)
4181 v.SetDefault("backups.databases.retention.daily", 0)
4182 v.SetDefault("backups.databases.retention.weekly", 0)
4183 v.SetDefault("backups.databases.retention.monthly", 0)
4184 v.SetDefault("backups.volumes.enabled", false)
4185 v.SetDefault("backups.volumes.interval", "24h")
4186 v.SetDefault("backups.volumes.compression", string(domain.VolumeBackupCompressionGzip))
4187 v.SetDefault("backups.volumes.timeout", "2h")
4188 v.SetDefault("backups.volumes.max_concurrency", 2)
4189 v.SetDefault("backups.volumes.helper_image", "alpine:3.20")
4190 v.SetDefault("backups.volumes.s3.bucket", "")
4191 v.SetDefault("backups.volumes.s3.region", "")
4192 v.SetDefault("backups.volumes.s3.prefix", "")
4193 v.SetDefault("backups.volumes.s3.endpoint", "")
4194 v.SetDefault("backups.volumes.s3.path_style", false)
4195 v.SetDefault("backups.volumes.s3.sse_algorithm", "")
4196 v.SetDefault("backups.volumes.s3.sse_kms_key_id", "")
4197 v.SetDefault("backups.volumes.retention.keep", 14)
4198 v.SetDefault("images.allowed_registries", []string{})
4199 v.SetDefault("images.require_digest", false)
4200 v.SetDefault("images.prune.enabled", false)
4201 v.SetDefault("images.prune.schedule", string(domain.ScheduleDaily))
4202 v.SetDefault("images.prune.keep_last", domain.DefaultImagePruneKeepLast)
4203 v.SetDefault("containers.security_profile", "compat")
4204 v.SetDefault("telemetry.enabled", false)
4205 v.SetDefault("telemetry.endpoint", "")
4206 v.SetDefault("telemetry.auth_token", "")
4207 v.SetDefault("telemetry.traces", true)
4208 v.SetDefault("telemetry.metrics", true)
4209 v.SetDefault("telemetry.logs", true)
4210 v.SetDefault("telemetry.trace_sample_rate", 1.0)
4211
4212 v.SetDefault("server.max_concurrent_connections", -1) // -1 = use default (10000), 0 = no limit
4213 v.SetDefault("server.registry_allowed_ips", []string{})
4214 v.SetDefault("server.proxy_allowed_ips", []string{})
4215 v.SetDefault("server.registry_listen_address", "")
4216
4217 ConfigureViper(v, configPath)
4218
4219 if err := v.ReadInConfig(); err != nil {
4220 if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
4221 return fmt.Errorf("failed to read config file: %w", err)
4222 }
4223 }
4224
4225 v.SetEnvPrefix("GORDON")
4226 v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
4227 v.AutomaticEnv()
4228
4229 return nil
4230}