fix(deploy): wire drain delay, prevent duplicate invalidation, and update docs
- commit
- 58a3ffa06f827804403e20ecc9d81707800e40db
- author
- bnema · · 2026-02-07 20:52 UTC
- parent
- 2c5325e
- signature
- unsigned
5 files · +40 -17
M docs/config/deploy.md +13 -0
| @@ -8,6 +8,7 @@ Controls how Gordon pulls images during deployment. | |||
| 8 | 8 | [deploy] | |
| 9 | 9 | pull_policy = "if-tag-changed" | |
| 10 | 10 | readiness_delay = "5s" | |
| 11 | + | drain_delay = "2s" | |
| 11 | 12 | ``` | |
| 12 | 13 | ||
| 13 | 14 | ## Settings | |
| @@ -33,7 +34,19 @@ considered ready for traffic. | |||
| 33 | 34 | ||
| 34 | 35 | Default: `"5s"` | |
| 35 | 36 | ||
| 37 | + | ### `deploy.drain_delay` | |
| 38 | + | ||
| 39 | + | How long Gordon waits after synchronous proxy cache invalidation before stopping | |
| 40 | + | the previous container during zero-downtime replacement. | |
| 41 | + | ||
| 42 | + | - Uses Go duration format (examples: `"2s"`, `"10s"`, `"1m"`). | |
| 43 | + | - Applied only when a previous container exists and cache invalidation was | |
| 44 | + | triggered for the deployed domain. | |
| 45 | + | ||
| 46 | + | Default: `"2s"` | |
| 47 | + | ||
| 36 | 48 | ## Notes | |
| 37 | 49 | ||
| 38 | 50 | - Changes to `deploy.pull_policy` require a restart to take effect. | |
| 39 | 51 | - Changes to `deploy.readiness_delay` require a restart to take effect. | |
| 52 | + | - Changes to `deploy.drain_delay` require a restart to take effect. | |
| @@ -8,6 +8,7 @@ Controls how Gordon pulls images during deployment. | |||||
| 8 | [deploy] | 8 | [deploy] | ||
| 9 | pull_policy = "if-tag-changed" | 9 | pull_policy = "if-tag-changed" | ||
| 10 | readiness_delay = "5s" | 10 | readiness_delay = "5s" | ||
| 11 | + | drain_delay = "2s" | |||
| 11 | ``` | 12 | ``` | ||
| 12 | 13 | ||||
| 13 | ## Settings | 14 | ## Settings | ||
| @@ -33,7 +34,19 @@ considered ready for traffic. | |||||
| 33 | 34 | ||||
| 34 | Default: `"5s"` | 35 | Default: `"5s"` | ||
| 35 | 36 | ||||
| 37 | + | ### `deploy.drain_delay` | |||
| 38 | + | ||||
| 39 | + | How long Gordon waits after synchronous proxy cache invalidation before stopping | |||
| 40 | + | the previous container during zero-downtime replacement. | |||
| 41 | + | ||||
| 42 | + | - Uses Go duration format (examples: `"2s"`, `"10s"`, `"1m"`). | |||
| 43 | + | - Applied only when a previous container exists and cache invalidation was | |||
| 44 | + | triggered for the deployed domain. | |||
| 45 | + | ||||
| 46 | + | Default: `"2s"` | |||
| 47 | + | ||||
| 36 | ## Notes | 48 | ## Notes | ||
| 37 | 49 | ||||
| 38 | - Changes to `deploy.pull_policy` require a restart to take effect. | 50 | - Changes to `deploy.pull_policy` require a restart to take effect. | ||
| 39 | - Changes to `deploy.readiness_delay` require a restart to take effect. | 51 | - Changes to `deploy.readiness_delay` require a restart to take effect. | ||
| 52 | + | - Changes to `deploy.drain_delay` require a restart to take effect. | |||
M docs/config/index.md +4 -0
| @@ -56,6 +56,8 @@ trusted_proxies = [] # IPs/CIDRs trusted for X-Forwarded-For | |||
| 56 | 56 | # Deploy behavior | |
| 57 | 57 | [deploy] | |
| 58 | 58 | pull_policy = "if-tag-changed" # always, if-not-present, if-tag-changed | |
| 59 | + | readiness_delay = "5s" # Wait after running before ready | |
| 60 | + | drain_delay = "2s" # Wait after proxy invalidation before old stop | |
| 59 | 61 | ||
| 60 | 62 | # Logging | |
| 61 | 63 | [logging] | |
| @@ -158,6 +160,7 @@ monthly = 12 | |||
| 158 | 160 | | `api.rate_limit.trusted_proxies` | `[]` | | |
| 159 | 161 | | `deploy.pull_policy` | `"if-tag-changed"` | | |
| 160 | 162 | | `deploy.readiness_delay` | `"5s"` | | |
| 163 | + | | `deploy.drain_delay` | `"2s"` | | |
| 161 | 164 | | `logging.level` | `"info"` | | |
| 162 | 165 | | `logging.format` | `"console"` | | |
| 163 | 166 | | `logging.file.enabled` | `false` | | |
| @@ -188,6 +191,7 @@ The following settings require a restart to take effect: | |||
| 188 | 191 | - `auth` settings | |
| 189 | 192 | - `deploy.pull_policy` | |
| 190 | 193 | - `deploy.readiness_delay` | |
| 194 | + | - `deploy.drain_delay` | |
| 191 | 195 | ||
| 192 | 196 | ## Environment Variable Override | |
| 193 | 197 | ||
| @@ -56,6 +56,8 @@ trusted_proxies = [] # IPs/CIDRs trusted for X-Forwarded-For | |||||
| 56 | # Deploy behavior | 56 | # Deploy behavior | ||
| 57 | [deploy] | 57 | [deploy] | ||
| 58 | pull_policy = "if-tag-changed" # always, if-not-present, if-tag-changed | 58 | pull_policy = "if-tag-changed" # always, if-not-present, if-tag-changed | ||
| 59 | + | readiness_delay = "5s" # Wait after running before ready | |||
| 60 | + | drain_delay = "2s" # Wait after proxy invalidation before old stop | |||
| 59 | 61 | ||||
| 60 | # Logging | 62 | # Logging | ||
| 61 | [logging] | 63 | [logging] | ||
| @@ -158,6 +160,7 @@ monthly = 12 | |||||
| 158 | | `api.rate_limit.trusted_proxies` | `[]` | | 160 | | `api.rate_limit.trusted_proxies` | `[]` | | ||
| 159 | | `deploy.pull_policy` | `"if-tag-changed"` | | 161 | | `deploy.pull_policy` | `"if-tag-changed"` | | ||
| 160 | | `deploy.readiness_delay` | `"5s"` | | 162 | | `deploy.readiness_delay` | `"5s"` | | ||
| 163 | + | | `deploy.drain_delay` | `"2s"` | | |||
| 161 | | `logging.level` | `"info"` | | 164 | | `logging.level` | `"info"` | | ||
| 162 | | `logging.format` | `"console"` | | 165 | | `logging.format` | `"console"` | | ||
| 163 | | `logging.file.enabled` | `false` | | 166 | | `logging.file.enabled` | `false` | | ||
| @@ -188,6 +191,7 @@ The following settings require a restart to take effect: | |||||
| 188 | - `auth` settings | 191 | - `auth` settings | ||
| 189 | - `deploy.pull_policy` | 192 | - `deploy.pull_policy` | ||
| 190 | - `deploy.readiness_delay` | 193 | - `deploy.readiness_delay` | ||
| 194 | + | - `deploy.drain_delay` | |||
| 191 | 195 | ||||
| 192 | ## Environment Variable Override | 196 | ## Environment Variable Override | ||
| 193 | 197 | ||||
M docs/config/reference.md +3 -0
| @@ -74,6 +74,8 @@ dir = "" # Env files directory (default: {da | |||
| 74 | 74 | # ============================================================================= | |
| 75 | 75 | [deploy] | |
| 76 | 76 | pull_policy = "if-tag-changed" # "always", "if-tag-changed", "never" | |
| 77 | + | readiness_delay = "5s" # Wait after running before considered ready | |
| 78 | + | drain_delay = "2s" # Wait after cache invalidation before old stop | |
| 77 | 79 | ||
| 78 | 80 | # ============================================================================= | |
| 79 | 81 | # AUTO-ROUTE | |
| @@ -173,6 +175,7 @@ monthly = 0 # Keep N monthly backups per DB | |||
| 173 | 175 | | `logging.container_logs.max_age` | `28` | 28 days | | |
| 174 | 176 | | `deploy.pull_policy` | `"if-tag-changed"` | Pull on tag change | | |
| 175 | 177 | | `deploy.readiness_delay` | `"5s"` | Delay before container is considered ready | | |
| 178 | + | | `deploy.drain_delay` | `"2s"` | Delay before stopping previous container after cache invalidation | | |
| 176 | 179 | | `auto_route.enabled` | `false` | Auto-route disabled | | |
| 177 | 180 | | `network_isolation.enabled` | `false` | Isolation disabled | | |
| 178 | 181 | | `network_isolation.network_prefix` | `"gordon"` | Network prefix | | |
| @@ -74,6 +74,8 @@ dir = "" # Env files directory (default: {da | |||||
| 74 | # ============================================================================= | 74 | # ============================================================================= | ||
| 75 | [deploy] | 75 | [deploy] | ||
| 76 | pull_policy = "if-tag-changed" # "always", "if-tag-changed", "never" | 76 | pull_policy = "if-tag-changed" # "always", "if-tag-changed", "never" | ||
| 77 | + | readiness_delay = "5s" # Wait after running before considered ready | |||
| 78 | + | drain_delay = "2s" # Wait after cache invalidation before old stop | |||
| 77 | 79 | ||||
| 78 | # ============================================================================= | 80 | # ============================================================================= | ||
| 79 | # AUTO-ROUTE | 81 | # AUTO-ROUTE | ||
| @@ -173,6 +175,7 @@ monthly = 0 # Keep N monthly backups per DB | |||||
| 173 | | `logging.container_logs.max_age` | `28` | 28 days | | 175 | | `logging.container_logs.max_age` | `28` | 28 days | | ||
| 174 | | `deploy.pull_policy` | `"if-tag-changed"` | Pull on tag change | | 176 | | `deploy.pull_policy` | `"if-tag-changed"` | Pull on tag change | | ||
| 175 | | `deploy.readiness_delay` | `"5s"` | Delay before container is considered ready | | 177 | | `deploy.readiness_delay` | `"5s"` | Delay before container is considered ready | | ||
| 178 | + | | `deploy.drain_delay` | `"2s"` | Delay before stopping previous container after cache invalidation | | |||
| 176 | | `auto_route.enabled` | `false` | Auto-route disabled | | 179 | | `auto_route.enabled` | `false` | Auto-route disabled | | ||
| 177 | | `network_isolation.enabled` | `false` | Isolation disabled | | 180 | | `network_isolation.enabled` | `false` | Isolation disabled | | ||
| 178 | | `network_isolation.network_prefix` | `"gordon"` | Network prefix | | 181 | | `network_isolation.network_prefix` | `"gordon"` | Network prefix | | ||
M internal/app/run.go +2 -6
| @@ -1059,6 +1059,7 @@ func createContainerService(ctx context.Context, v *viper.Viper, cfg Config, svc | |||
| 1059 | 1059 | NetworkGroups: svc.configSvc.GetNetworkGroups(), | |
| 1060 | 1060 | Attachments: svc.configSvc.GetAttachments(), | |
| 1061 | 1061 | ReadinessDelay: v.GetDuration("deploy.readiness_delay"), | |
| 1062 | + | DrainDelay: v.GetDuration("deploy.drain_delay"), | |
| 1062 | 1063 | } | |
| 1063 | 1064 | ||
| 1064 | 1065 | if cfg.Auth.Enabled && svc.authSvc != nil { | |
| @@ -1170,12 +1171,6 @@ func registerEventHandlers(ctx context.Context, svc *services, cfg Config) error | |||
| 1170 | 1171 | return fmt.Errorf("failed to subscribe manual deploy handler: %w", err) | |
| 1171 | 1172 | } | |
| 1172 | 1173 | ||
| 1173 | - | // Proxy cache invalidation on container deployment (for zero-downtime) | |
| 1174 | - | containerDeployedHandler := proxy.NewContainerDeployedHandler(ctx, svc.proxySvc) | |
| 1175 | - | if err := svc.eventBus.Subscribe(containerDeployedHandler); err != nil { | |
| 1176 | - | return fmt.Errorf("failed to subscribe container deployed handler: %w", err) | |
| 1177 | - | } | |
| 1178 | - | ||
| 1179 | 1174 | // Proxy cache invalidation on config reload (clears stale targets for removed routes) | |
| 1180 | 1175 | configReloadProxyHandler := proxy.NewConfigReloadProxyHandler(ctx, svc.proxySvc) | |
| 1181 | 1176 | if err := svc.eventBus.Subscribe(configReloadProxyHandler); err != nil { | |
| @@ -1860,6 +1855,7 @@ func loadConfig(v *viper.Viper, configPath string) error { | |||
| 1860 | 1855 | v.SetDefault("server.max_concurrent_connections", -1) // -1 = use default (10000), 0 = no limit | |
| 1861 | 1856 | v.SetDefault("server.registry_allowed_ips", []string{}) | |
| 1862 | 1857 | v.SetDefault("deploy.readiness_delay", "5s") | |
| 1858 | + | v.SetDefault("deploy.drain_delay", "2s") | |
| 1863 | 1859 | ||
| 1864 | 1860 | ConfigureViper(v, configPath) | |
| 1865 | 1861 | ||
| @@ -1059,6 +1059,7 @@ func createContainerService(ctx context.Context, v *viper.Viper, cfg Config, svc | |||||
| 1059 | NetworkGroups: svc.configSvc.GetNetworkGroups(), | 1059 | NetworkGroups: svc.configSvc.GetNetworkGroups(), | ||
| 1060 | Attachments: svc.configSvc.GetAttachments(), | 1060 | Attachments: svc.configSvc.GetAttachments(), | ||
| 1061 | ReadinessDelay: v.GetDuration("deploy.readiness_delay"), | 1061 | ReadinessDelay: v.GetDuration("deploy.readiness_delay"), | ||
| 1062 | + | DrainDelay: v.GetDuration("deploy.drain_delay"), | |||
| 1062 | } | 1063 | } | ||
| 1063 | 1064 | ||||
| 1064 | if cfg.Auth.Enabled && svc.authSvc != nil { | 1065 | if cfg.Auth.Enabled && svc.authSvc != nil { | ||
| @@ -1170,12 +1171,6 @@ func registerEventHandlers(ctx context.Context, svc *services, cfg Config) error | |||||
| 1170 | return fmt.Errorf("failed to subscribe manual deploy handler: %w", err) | 1171 | return fmt.Errorf("failed to subscribe manual deploy handler: %w", err) | ||
| 1171 | } | 1172 | } | ||
| 1172 | 1173 | ||||
| 1173 | - | // Proxy cache invalidation on container deployment (for zero-downtime) | |||
| 1174 | - | containerDeployedHandler := proxy.NewContainerDeployedHandler(ctx, svc.proxySvc) | |||
| 1175 | - | if err := svc.eventBus.Subscribe(containerDeployedHandler); err != nil { | |||
| 1176 | - | return fmt.Errorf("failed to subscribe container deployed handler: %w", err) | |||
| 1177 | - | } | |||
| 1178 | - | ||||
| 1179 | // Proxy cache invalidation on config reload (clears stale targets for removed routes) | 1174 | // Proxy cache invalidation on config reload (clears stale targets for removed routes) | ||
| 1180 | configReloadProxyHandler := proxy.NewConfigReloadProxyHandler(ctx, svc.proxySvc) | 1175 | configReloadProxyHandler := proxy.NewConfigReloadProxyHandler(ctx, svc.proxySvc) | ||
| 1181 | if err := svc.eventBus.Subscribe(configReloadProxyHandler); err != nil { | 1176 | if err := svc.eventBus.Subscribe(configReloadProxyHandler); err != nil { | ||
| @@ -1860,6 +1855,7 @@ func loadConfig(v *viper.Viper, configPath string) error { | |||||
| 1860 | v.SetDefault("server.max_concurrent_connections", -1) // -1 = use default (10000), 0 = no limit | 1855 | v.SetDefault("server.max_concurrent_connections", -1) // -1 = use default (10000), 0 = no limit | ||
| 1861 | v.SetDefault("server.registry_allowed_ips", []string{}) | 1856 | v.SetDefault("server.registry_allowed_ips", []string{}) | ||
| 1862 | v.SetDefault("deploy.readiness_delay", "5s") | 1857 | v.SetDefault("deploy.readiness_delay", "5s") | ||
| 1858 | + | v.SetDefault("deploy.drain_delay", "2s") | |||
| 1863 | 1859 | ||||
| 1864 | ConfigureViper(v, configPath) | 1860 | ConfigureViper(v, configPath) | ||
| 1865 | 1861 | ||||
M internal/usecase/container/service.go +18 -11
| @@ -229,8 +229,8 @@ func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Conta | |||
| 229 | 229 | return nil, err | |
| 230 | 230 | } | |
| 231 | 231 | ||
| 232 | - | s.activateDeployedContainer(ctx, route.Domain, newContainer) | |
| 233 | - | s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, newContainer.ID) | |
| 232 | + | invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer) | |
| 233 | + | s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, invalidated, newContainer.ID) | |
| 234 | 234 | ||
| 235 | 235 | // Start container log collection (non-blocking, errors don't fail deployment) | |
| 236 | 236 | s.startLogCollection(ctx, newContainer.ID, route.Domain) | |
| @@ -346,7 +346,7 @@ func (s *Service) createStartedContainer(ctx context.Context, route domain.Route | |||
| 346 | 346 | return inspected, nil | |
| 347 | 347 | } | |
| 348 | 348 | ||
| 349 | - | func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) { | |
| 349 | + | func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool { | |
| 350 | 350 | s.mu.Lock() | |
| 351 | 351 | s.containers[domainName] = container | |
| 352 | 352 | s.mu.Unlock() | |
| @@ -358,22 +358,29 @@ func (s *Service) activateDeployedContainer(ctx context.Context, domainName stri | |||
| 358 | 358 | s.mu.RUnlock() | |
| 359 | 359 | if inv != nil { | |
| 360 | 360 | inv.InvalidateTarget(ctx, domainName) | |
| 361 | + | return true | |
| 361 | 362 | } | |
| 363 | + | ||
| 364 | + | return false | |
| 362 | 365 | } | |
| 363 | 366 | ||
| 364 | - | func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting bool, newContainerID string) { | |
| 367 | + | func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) { | |
| 365 | 368 | if !hasExisting { | |
| 366 | 369 | return | |
| 367 | 370 | } | |
| 368 | 371 | log := zerowrap.FromCtx(ctx) | |
| 369 | 372 | ||
| 370 | - | drainDelay := s.config.DrainDelay | |
| 371 | - | if drainDelay == 0 { | |
| 372 | - | drainDelay = 2 * time.Second | |
| 373 | - | } | |
| 374 | - | select { | |
| 375 | - | case <-time.After(drainDelay): | |
| 376 | - | case <-ctx.Done(): | |
| 373 | + | if invalidated { | |
| 374 | + | s.mu.RLock() | |
| 375 | + | drainDelay := s.config.DrainDelay | |
| 376 | + | s.mu.RUnlock() | |
| 377 | + | if drainDelay == 0 { | |
| 378 | + | drainDelay = 2 * time.Second | |
| 379 | + | } | |
| 380 | + | select { | |
| 381 | + | case <-time.After(drainDelay): | |
| 382 | + | case <-ctx.Done(): | |
| 383 | + | } | |
| 377 | 384 | } | |
| 378 | 385 | ||
| 379 | 386 | if err := ctx.Err(); err != nil { | |
| @@ -229,8 +229,8 @@ func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Conta | |||||
| 229 | return nil, err | 229 | return nil, err | ||
| 230 | } | 230 | } | ||
| 231 | 231 | ||||
| 232 | - | s.activateDeployedContainer(ctx, route.Domain, newContainer) | 232 | + | invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer) |
| 233 | - | s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, newContainer.ID) | 233 | + | s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, invalidated, newContainer.ID) |
| 234 | 234 | ||||
| 235 | // Start container log collection (non-blocking, errors don't fail deployment) | 235 | // Start container log collection (non-blocking, errors don't fail deployment) | ||
| 236 | s.startLogCollection(ctx, newContainer.ID, route.Domain) | 236 | s.startLogCollection(ctx, newContainer.ID, route.Domain) | ||
| @@ -346,7 +346,7 @@ func (s *Service) createStartedContainer(ctx context.Context, route domain.Route | |||||
| 346 | return inspected, nil | 346 | return inspected, nil | ||
| 347 | } | 347 | } | ||
| 348 | 348 | ||||
| 349 | - | func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) { | 349 | + | func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool { |
| 350 | s.mu.Lock() | 350 | s.mu.Lock() | ||
| 351 | s.containers[domainName] = container | 351 | s.containers[domainName] = container | ||
| 352 | s.mu.Unlock() | 352 | s.mu.Unlock() | ||
| @@ -358,22 +358,29 @@ func (s *Service) activateDeployedContainer(ctx context.Context, domainName stri | |||||
| 358 | s.mu.RUnlock() | 358 | s.mu.RUnlock() | ||
| 359 | if inv != nil { | 359 | if inv != nil { | ||
| 360 | inv.InvalidateTarget(ctx, domainName) | 360 | inv.InvalidateTarget(ctx, domainName) | ||
| 361 | + | return true | |||
| 361 | } | 362 | } | ||
| 363 | + | ||||
| 364 | + | return false | |||
| 362 | } | 365 | } | ||
| 363 | 366 | ||||
| 364 | - | func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting bool, newContainerID string) { | 367 | + | func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) { |
| 365 | if !hasExisting { | 368 | if !hasExisting { | ||
| 366 | return | 369 | return | ||
| 367 | } | 370 | } | ||
| 368 | log := zerowrap.FromCtx(ctx) | 371 | log := zerowrap.FromCtx(ctx) | ||
| 369 | 372 | ||||
| 370 | - | drainDelay := s.config.DrainDelay | 373 | + | if invalidated { |
| 371 | - | if drainDelay == 0 { | 374 | + | s.mu.RLock() |
| 372 | - | drainDelay = 2 * time.Second | 375 | + | drainDelay := s.config.DrainDelay |
| 373 | - | } | 376 | + | s.mu.RUnlock() |
| 374 | - | select { | 377 | + | if drainDelay == 0 { |
| 375 | - | case <-time.After(drainDelay): | 378 | + | drainDelay = 2 * time.Second |
| 376 | - | case <-ctx.Done(): | 379 | + | } |
| 380 | + | select { | |||
| 381 | + | case <-time.After(drainDelay): | |||
| 382 | + | case <-ctx.Done(): | |||
| 383 | + | } | |||
| 377 | } | 384 | } | ||
| 378 | 385 | ||||
| 379 | if err := ctx.Err(); err != nil { | 386 | if err := ctx.Err(); err != nil { | ||