detached 4fba121

fix(deploy): wire drain delay, prevent duplicate invalidation, and update docs

commit
4fba1212b38979c37049bdcddb4dfeb51b2c311d
author
bnema · · 2026-02-07 20:52 UTC
parent
2632e63
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
5 files · +40 -17
M docs/config/deploy.md +13 -0
@@ -8,6 +8,7 @@ Controls how Gordon pulls images during deployment.
88 [deploy]
99 pull_policy = "if-tag-changed"
1010 readiness_delay = "5s"
11+drain_delay = "2s"
1112 ```
1213
1314 ## Settings
@@ -33,7 +34,19 @@ considered ready for traffic.
3334
3435 Default: `"5s"`
3536
37+### `deploy.drain_delay`
38+
39+How long Gordon waits after synchronous proxy cache invalidation before stopping
40+the previous container during zero-downtime replacement.
41+
42+- Uses Go duration format (examples: `"2s"`, `"10s"`, `"1m"`).
43+- Applied only when a previous container exists and cache invalidation was
44+ triggered for the deployed domain.
45+
46+Default: `"2s"`
47+
3648 ## Notes
3749
3850 - Changes to `deploy.pull_policy` require a restart to take effect.
3951 - Changes to `deploy.readiness_delay` require a restart to take effect.
52+- Changes to `deploy.drain_delay` require a restart to take effect.
@@ -8,6 +8,7 @@ Controls how Gordon pulls images during deployment.
8 [deploy]8 [deploy]
9 pull_policy = "if-tag-changed"9 pull_policy = "if-tag-changed"
10 readiness_delay = "5s"10 readiness_delay = "5s"
11+drain_delay = "2s"
11 ```12 ```
12 13
13 ## Settings14 ## Settings
@@ -33,7 +34,19 @@ considered ready for traffic.
33 34
34 Default: `"5s"`35 Default: `"5s"`
35 36
37+### `deploy.drain_delay`
38+
39+How long Gordon waits after synchronous proxy cache invalidation before stopping
40+the previous container during zero-downtime replacement.
41+
42+- Uses Go duration format (examples: `"2s"`, `"10s"`, `"1m"`).
43+- Applied only when a previous container exists and cache invalidation was
44+ triggered for the deployed domain.
45+
46+Default: `"2s"`
47+
36 ## Notes48 ## Notes
37 49
38 - Changes to `deploy.pull_policy` require a restart to take effect.50 - Changes to `deploy.pull_policy` require a restart to take effect.
39 - Changes to `deploy.readiness_delay` require a restart to take effect.51 - Changes to `deploy.readiness_delay` require a restart to take effect.
52+- Changes to `deploy.drain_delay` require a restart to take effect.
M docs/config/index.md +4 -0
@@ -56,6 +56,8 @@ trusted_proxies = [] # IPs/CIDRs trusted for X-Forwarded-For
5656 # Deploy behavior
5757 [deploy]
5858 pull_policy = "if-tag-changed" # always, if-not-present, if-tag-changed
59+readiness_delay = "5s" # Wait after running before ready
60+drain_delay = "2s" # Wait after proxy invalidation before old stop
5961
6062 # Logging
6163 [logging]
@@ -158,6 +160,7 @@ monthly = 12
158160 | `api.rate_limit.trusted_proxies` | `[]` |
159161 | `deploy.pull_policy` | `"if-tag-changed"` |
160162 | `deploy.readiness_delay` | `"5s"` |
163+| `deploy.drain_delay` | `"2s"` |
161164 | `logging.level` | `"info"` |
162165 | `logging.format` | `"console"` |
163166 | `logging.file.enabled` | `false` |
@@ -188,6 +191,7 @@ The following settings require a restart to take effect:
188191 - `auth` settings
189192 - `deploy.pull_policy`
190193 - `deploy.readiness_delay`
194+- `deploy.drain_delay`
191195
192196 ## Environment Variable Override
193197
@@ -56,6 +56,8 @@ trusted_proxies = [] # IPs/CIDRs trusted for X-Forwarded-For
56 # Deploy behavior56 # Deploy behavior
57 [deploy]57 [deploy]
58 pull_policy = "if-tag-changed" # always, if-not-present, if-tag-changed58 pull_policy = "if-tag-changed" # always, if-not-present, if-tag-changed
59+readiness_delay = "5s" # Wait after running before ready
60+drain_delay = "2s" # Wait after proxy invalidation before old stop
59 61
60 # Logging62 # Logging
61 [logging]63 [logging]
@@ -158,6 +160,7 @@ monthly = 12
158 | `api.rate_limit.trusted_proxies` | `[]` |160 | `api.rate_limit.trusted_proxies` | `[]` |
159 | `deploy.pull_policy` | `"if-tag-changed"` |161 | `deploy.pull_policy` | `"if-tag-changed"` |
160 | `deploy.readiness_delay` | `"5s"` |162 | `deploy.readiness_delay` | `"5s"` |
163+| `deploy.drain_delay` | `"2s"` |
161 | `logging.level` | `"info"` |164 | `logging.level` | `"info"` |
162 | `logging.format` | `"console"` |165 | `logging.format` | `"console"` |
163 | `logging.file.enabled` | `false` |166 | `logging.file.enabled` | `false` |
@@ -188,6 +191,7 @@ The following settings require a restart to take effect:
188 - `auth` settings191 - `auth` settings
189 - `deploy.pull_policy`192 - `deploy.pull_policy`
190 - `deploy.readiness_delay`193 - `deploy.readiness_delay`
194+- `deploy.drain_delay`
191 195
192 ## Environment Variable Override196 ## Environment Variable Override
193 197
M docs/config/reference.md +3 -0
@@ -74,6 +74,8 @@ dir = "" # Env files directory (default: {da
7474 # =============================================================================
7575 [deploy]
7676 pull_policy = "if-tag-changed" # "always", "if-tag-changed", "never"
77+readiness_delay = "5s" # Wait after running before considered ready
78+drain_delay = "2s" # Wait after cache invalidation before old stop
7779
7880 # =============================================================================
7981 # AUTO-ROUTE
@@ -173,6 +175,7 @@ monthly = 0 # Keep N monthly backups per DB
173175 | `logging.container_logs.max_age` | `28` | 28 days |
174176 | `deploy.pull_policy` | `"if-tag-changed"` | Pull on tag change |
175177 | `deploy.readiness_delay` | `"5s"` | Delay before container is considered ready |
178+| `deploy.drain_delay` | `"2s"` | Delay before stopping previous container after cache invalidation |
176179 | `auto_route.enabled` | `false` | Auto-route disabled |
177180 | `network_isolation.enabled` | `false` | Isolation disabled |
178181 | `network_isolation.network_prefix` | `"gordon"` | Network prefix |
@@ -74,6 +74,8 @@ dir = "" # Env files directory (default: {da
74 # =============================================================================74 # =============================================================================
75 [deploy]75 [deploy]
76 pull_policy = "if-tag-changed" # "always", "if-tag-changed", "never"76 pull_policy = "if-tag-changed" # "always", "if-tag-changed", "never"
77+readiness_delay = "5s" # Wait after running before considered ready
78+drain_delay = "2s" # Wait after cache invalidation before old stop
77 79
78 # =============================================================================80 # =============================================================================
79 # AUTO-ROUTE81 # AUTO-ROUTE
@@ -173,6 +175,7 @@ monthly = 0 # Keep N monthly backups per DB
173 | `logging.container_logs.max_age` | `28` | 28 days |175 | `logging.container_logs.max_age` | `28` | 28 days |
174 | `deploy.pull_policy` | `"if-tag-changed"` | Pull on tag change |176 | `deploy.pull_policy` | `"if-tag-changed"` | Pull on tag change |
175 | `deploy.readiness_delay` | `"5s"` | Delay before container is considered ready |177 | `deploy.readiness_delay` | `"5s"` | Delay before container is considered ready |
178+| `deploy.drain_delay` | `"2s"` | Delay before stopping previous container after cache invalidation |
176 | `auto_route.enabled` | `false` | Auto-route disabled |179 | `auto_route.enabled` | `false` | Auto-route disabled |
177 | `network_isolation.enabled` | `false` | Isolation disabled |180 | `network_isolation.enabled` | `false` | Isolation disabled |
178 | `network_isolation.network_prefix` | `"gordon"` | Network prefix |181 | `network_isolation.network_prefix` | `"gordon"` | Network prefix |
M internal/app/run.go +2 -6
@@ -1059,6 +1059,7 @@ func createContainerService(ctx context.Context, v *viper.Viper, cfg Config, svc
10591059 NetworkGroups: svc.configSvc.GetNetworkGroups(),
10601060 Attachments: svc.configSvc.GetAttachments(),
10611061 ReadinessDelay: v.GetDuration("deploy.readiness_delay"),
1062+ DrainDelay: v.GetDuration("deploy.drain_delay"),
10621063 }
10631064
10641065 if cfg.Auth.Enabled && svc.authSvc != nil {
@@ -1170,12 +1171,6 @@ func registerEventHandlers(ctx context.Context, svc *services, cfg Config) error
11701171 return fmt.Errorf("failed to subscribe manual deploy handler: %w", err)
11711172 }
11721173
1173- // Proxy cache invalidation on container deployment (for zero-downtime)
1174- containerDeployedHandler := proxy.NewContainerDeployedHandler(ctx, svc.proxySvc)
1175- if err := svc.eventBus.Subscribe(containerDeployedHandler); err != nil {
1176- return fmt.Errorf("failed to subscribe container deployed handler: %w", err)
1177- }
1178-
11791174 // Proxy cache invalidation on config reload (clears stale targets for removed routes)
11801175 configReloadProxyHandler := proxy.NewConfigReloadProxyHandler(ctx, svc.proxySvc)
11811176 if err := svc.eventBus.Subscribe(configReloadProxyHandler); err != nil {
@@ -1860,6 +1855,7 @@ func loadConfig(v *viper.Viper, configPath string) error {
18601855 v.SetDefault("server.max_concurrent_connections", -1) // -1 = use default (10000), 0 = no limit
18611856 v.SetDefault("server.registry_allowed_ips", []string{})
18621857 v.SetDefault("deploy.readiness_delay", "5s")
1858+ v.SetDefault("deploy.drain_delay", "2s")
18631859
18641860 ConfigureViper(v, configPath)
18651861
@@ -1059,6 +1059,7 @@ func createContainerService(ctx context.Context, v *viper.Viper, cfg Config, svc
1059 NetworkGroups: svc.configSvc.GetNetworkGroups(),1059 NetworkGroups: svc.configSvc.GetNetworkGroups(),
1060 Attachments: svc.configSvc.GetAttachments(),1060 Attachments: svc.configSvc.GetAttachments(),
1061 ReadinessDelay: v.GetDuration("deploy.readiness_delay"),1061 ReadinessDelay: v.GetDuration("deploy.readiness_delay"),
1062+ DrainDelay: v.GetDuration("deploy.drain_delay"),
1062 }1063 }
1063 1064
1064 if cfg.Auth.Enabled && svc.authSvc != nil {1065 if cfg.Auth.Enabled && svc.authSvc != nil {
@@ -1170,12 +1171,6 @@ func registerEventHandlers(ctx context.Context, svc *services, cfg Config) error
1170 return fmt.Errorf("failed to subscribe manual deploy handler: %w", err)1171 return fmt.Errorf("failed to subscribe manual deploy handler: %w", err)
1171 }1172 }
1172 1173
1173- // Proxy cache invalidation on container deployment (for zero-downtime)
1174- containerDeployedHandler := proxy.NewContainerDeployedHandler(ctx, svc.proxySvc)
1175- if err := svc.eventBus.Subscribe(containerDeployedHandler); err != nil {
1176- return fmt.Errorf("failed to subscribe container deployed handler: %w", err)
1177- }
1178-
1179 // Proxy cache invalidation on config reload (clears stale targets for removed routes)1174 // Proxy cache invalidation on config reload (clears stale targets for removed routes)
1180 configReloadProxyHandler := proxy.NewConfigReloadProxyHandler(ctx, svc.proxySvc)1175 configReloadProxyHandler := proxy.NewConfigReloadProxyHandler(ctx, svc.proxySvc)
1181 if err := svc.eventBus.Subscribe(configReloadProxyHandler); err != nil {1176 if err := svc.eventBus.Subscribe(configReloadProxyHandler); err != nil {
@@ -1860,6 +1855,7 @@ func loadConfig(v *viper.Viper, configPath string) error {
1860 v.SetDefault("server.max_concurrent_connections", -1) // -1 = use default (10000), 0 = no limit1855 v.SetDefault("server.max_concurrent_connections", -1) // -1 = use default (10000), 0 = no limit
1861 v.SetDefault("server.registry_allowed_ips", []string{})1856 v.SetDefault("server.registry_allowed_ips", []string{})
1862 v.SetDefault("deploy.readiness_delay", "5s")1857 v.SetDefault("deploy.readiness_delay", "5s")
1858+ v.SetDefault("deploy.drain_delay", "2s")
1863 1859
1864 ConfigureViper(v, configPath)1860 ConfigureViper(v, configPath)
1865 1861
M internal/usecase/container/service.go +18 -11
@@ -229,8 +229,8 @@ func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Conta
229229 return nil, err
230230 }
231231
232- s.activateDeployedContainer(ctx, route.Domain, newContainer)
233- s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, newContainer.ID)
232+ invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
233+ s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, invalidated, newContainer.ID)
234234
235235 // Start container log collection (non-blocking, errors don't fail deployment)
236236 s.startLogCollection(ctx, newContainer.ID, route.Domain)
@@ -346,7 +346,7 @@ func (s *Service) createStartedContainer(ctx context.Context, route domain.Route
346346 return inspected, nil
347347 }
348348
349-func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) {
349+func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
350350 s.mu.Lock()
351351 s.containers[domainName] = container
352352 s.mu.Unlock()
@@ -358,22 +358,29 @@ func (s *Service) activateDeployedContainer(ctx context.Context, domainName stri
358358 s.mu.RUnlock()
359359 if inv != nil {
360360 inv.InvalidateTarget(ctx, domainName)
361+ return true
361362 }
363+
364+ return false
362365 }
363366
364-func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting bool, newContainerID string) {
367+func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
365368 if !hasExisting {
366369 return
367370 }
368371 log := zerowrap.FromCtx(ctx)
369372
370- drainDelay := s.config.DrainDelay
371- if drainDelay == 0 {
372- drainDelay = 2 * time.Second
373- }
374- select {
375- case <-time.After(drainDelay):
376- case <-ctx.Done():
373+ if invalidated {
374+ s.mu.RLock()
375+ drainDelay := s.config.DrainDelay
376+ s.mu.RUnlock()
377+ if drainDelay == 0 {
378+ drainDelay = 2 * time.Second
379+ }
380+ select {
381+ case <-time.After(drainDelay):
382+ case <-ctx.Done():
383+ }
377384 }
378385
379386 if err := ctx.Err(); err != nil {
@@ -229,8 +229,8 @@ func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Conta
229 return nil, err229 return nil, err
230 }230 }
231 231
232- s.activateDeployedContainer(ctx, route.Domain, newContainer)232+ invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
233- s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, newContainer.ID)233+ s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, invalidated, newContainer.ID)
234 234
235 // Start container log collection (non-blocking, errors don't fail deployment)235 // Start container log collection (non-blocking, errors don't fail deployment)
236 s.startLogCollection(ctx, newContainer.ID, route.Domain)236 s.startLogCollection(ctx, newContainer.ID, route.Domain)
@@ -346,7 +346,7 @@ func (s *Service) createStartedContainer(ctx context.Context, route domain.Route
346 return inspected, nil346 return inspected, nil
347 }347 }
348 348
349-func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) {349+func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
350 s.mu.Lock()350 s.mu.Lock()
351 s.containers[domainName] = container351 s.containers[domainName] = container
352 s.mu.Unlock()352 s.mu.Unlock()
@@ -358,22 +358,29 @@ func (s *Service) activateDeployedContainer(ctx context.Context, domainName stri
358 s.mu.RUnlock()358 s.mu.RUnlock()
359 if inv != nil {359 if inv != nil {
360 inv.InvalidateTarget(ctx, domainName)360 inv.InvalidateTarget(ctx, domainName)
361+ return true
361 }362 }
363+
364+ return false
362 }365 }
363 366
364-func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting bool, newContainerID string) {367+func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
365 if !hasExisting {368 if !hasExisting {
366 return369 return
367 }370 }
368 log := zerowrap.FromCtx(ctx)371 log := zerowrap.FromCtx(ctx)
369 372
370- drainDelay := s.config.DrainDelay373+ if invalidated {
371- if drainDelay == 0 {374+ s.mu.RLock()
372- drainDelay = 2 * time.Second375+ drainDelay := s.config.DrainDelay
373- }376+ s.mu.RUnlock()
374- select {377+ if drainDelay == 0 {
375- case <-time.After(drainDelay):378+ drainDelay = 2 * time.Second
376- case <-ctx.Done():379+ }
380+ select {
381+ case <-time.After(drainDelay):
382+ case <-ctx.Done():
383+ }
377 }384 }
378 385
379 if err := ctx.Err(); err != nil {386 if err := ctx.Err(); err != nil {

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 7

Tags 106

↑↓ moveenter openesc close