feat: post-switch stabilization window with automatic rollback
After switching traffic to new container, Deploy monitors it briefly. If new container crashes during stabilization, traffic is switched back to old container and the failed new container is cleaned up.
- commit
- 0f1a5e42efb12f35bcf9fd855865cd9d04f63abe
- author
- brice · · 2026-03-02 16:40 UTC
- parent
- d95917d
- signature
- unsigned
2 files · +294 -38
M internal/usecase/container/service.go +64 -0
| @@ -48,6 +48,7 @@ type Config struct { | |||
| 48 | 48 | DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured | |
| 49 | 49 | DrainMode string // Drain strategy: auto, inflight, delay | |
| 50 | 50 | DrainTimeout time.Duration // Max wait for in-flight requests to drain | |
| 51 | + | StabilizationDelay time.Duration // Pause after traffic switch before verifying new container health | |
| 51 | 52 | } | |
| 52 | 53 | ||
| 53 | 54 | var tracer = otel.Tracer("gordon.container") | |
| @@ -305,6 +306,15 @@ func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Conta | |||
| 305 | 306 | } | |
| 306 | 307 | ||
| 307 | 308 | invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer) | |
| 309 | + | ||
| 310 | + | // Post-switch stabilization: verify new container stays running | |
| 311 | + | if hasExisting { | |
| 312 | + | if !s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing) { | |
| 313 | + | // Rollback performed — old container is restored | |
| 314 | + | return existing, nil | |
| 315 | + | } | |
| 316 | + | } | |
| 317 | + | ||
| 308 | 318 | s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, invalidated, newContainer.ID) | |
| 309 | 319 | ||
| 310 | 320 | // Start container log collection (non-blocking, errors don't fail deployment) | |
| @@ -549,6 +559,60 @@ func (s *Service) activateDeployedContainer(ctx context.Context, domainName stri | |||
| 549 | 559 | return false | |
| 550 | 560 | } | |
| 551 | 561 | ||
| 562 | + | // stabilizeNewContainer monitors the new container briefly after traffic switch. | |
| 563 | + | // If it crashes during this window, rolls back to old container. | |
| 564 | + | // Returns true if stabilization succeeded, false if rollback was performed. | |
| 565 | + | func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) bool { | |
| 566 | + | log := zerowrap.FromCtx(ctx) | |
| 567 | + | ||
| 568 | + | if oldContainer == nil { | |
| 569 | + | return true | |
| 570 | + | } | |
| 571 | + | ||
| 572 | + | s.mu.RLock() | |
| 573 | + | delay := s.config.StabilizationDelay | |
| 574 | + | s.mu.RUnlock() | |
| 575 | + | if delay == 0 { | |
| 576 | + | delay = 2 * time.Second | |
| 577 | + | } | |
| 578 | + | ||
| 579 | + | // Brief stabilization: verify new container is still running after delay | |
| 580 | + | select { | |
| 581 | + | case <-time.After(delay): | |
| 582 | + | case <-ctx.Done(): | |
| 583 | + | return true | |
| 584 | + | } | |
| 585 | + | ||
| 586 | + | running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID) | |
| 587 | + | if err != nil || !running { | |
| 588 | + | log.Error(). | |
| 589 | + | Str("new_container", newContainer.ID). | |
| 590 | + | Str("old_container", oldContainer.ID). | |
| 591 | + | Msg("new container crashed during stabilization, rolling back to old") | |
| 592 | + | ||
| 593 | + | // Rollback: restore old container as tracked | |
| 594 | + | s.mu.Lock() | |
| 595 | + | s.containers[domainName] = oldContainer | |
| 596 | + | s.mu.Unlock() | |
| 597 | + | ||
| 598 | + | // Re-invalidate proxy cache to point back to old | |
| 599 | + | s.mu.RLock() | |
| 600 | + | inv := s.cacheInvalidator | |
| 601 | + | s.mu.RUnlock() | |
| 602 | + | if inv != nil { | |
| 603 | + | inv.InvalidateTarget(ctx, domainName) | |
| 604 | + | } | |
| 605 | + | ||
| 606 | + | // Cleanup failed new container | |
| 607 | + | _ = s.runtime.StopContainer(ctx, newContainer.ID) | |
| 608 | + | _ = s.runtime.RemoveContainer(ctx, newContainer.ID, true) | |
| 609 | + | ||
| 610 | + | return false | |
| 611 | + | } | |
| 612 | + | ||
| 613 | + | return true | |
| 614 | + | } | |
| 615 | + | ||
| 552 | 616 | func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) { | |
| 553 | 617 | if !hasExisting { | |
| 554 | 618 | return | |
| @@ -48,6 +48,7 @@ type Config struct { | |||||
| 48 | DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured | 48 | DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured | ||
| 49 | DrainMode string // Drain strategy: auto, inflight, delay | 49 | DrainMode string // Drain strategy: auto, inflight, delay | ||
| 50 | DrainTimeout time.Duration // Max wait for in-flight requests to drain | 50 | DrainTimeout time.Duration // Max wait for in-flight requests to drain | ||
| 51 | + | StabilizationDelay time.Duration // Pause after traffic switch before verifying new container health | |||
| 51 | } | 52 | } | ||
| 52 | 53 | ||||
| 53 | var tracer = otel.Tracer("gordon.container") | 54 | var tracer = otel.Tracer("gordon.container") | ||
| @@ -305,6 +306,15 @@ func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Conta | |||||
| 305 | } | 306 | } | ||
| 306 | 307 | ||||
| 307 | invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer) | 308 | invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer) | ||
| 309 | + | ||||
| 310 | + | // Post-switch stabilization: verify new container stays running | |||
| 311 | + | if hasExisting { | |||
| 312 | + | if !s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing) { | |||
| 313 | + | // Rollback performed — old container is restored | |||
| 314 | + | return existing, nil | |||
| 315 | + | } | |||
| 316 | + | } | |||
| 317 | + | ||||
| 308 | s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, invalidated, newContainer.ID) | 318 | s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, invalidated, newContainer.ID) | ||
| 309 | 319 | ||||
| 310 | // Start container log collection (non-blocking, errors don't fail deployment) | 320 | // Start container log collection (non-blocking, errors don't fail deployment) | ||
| @@ -549,6 +559,60 @@ func (s *Service) activateDeployedContainer(ctx context.Context, domainName stri | |||||
| 549 | return false | 559 | return false | ||
| 550 | } | 560 | } | ||
| 551 | 561 | ||||
| 562 | + | // stabilizeNewContainer monitors the new container briefly after traffic switch. | |||
| 563 | + | // If it crashes during this window, rolls back to old container. | |||
| 564 | + | // Returns true if stabilization succeeded, false if rollback was performed. | |||
| 565 | + | func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) bool { | |||
| 566 | + | log := zerowrap.FromCtx(ctx) | |||
| 567 | + | ||||
| 568 | + | if oldContainer == nil { | |||
| 569 | + | return true | |||
| 570 | + | } | |||
| 571 | + | ||||
| 572 | + | s.mu.RLock() | |||
| 573 | + | delay := s.config.StabilizationDelay | |||
| 574 | + | s.mu.RUnlock() | |||
| 575 | + | if delay == 0 { | |||
| 576 | + | delay = 2 * time.Second | |||
| 577 | + | } | |||
| 578 | + | ||||
| 579 | + | // Brief stabilization: verify new container is still running after delay | |||
| 580 | + | select { | |||
| 581 | + | case <-time.After(delay): | |||
| 582 | + | case <-ctx.Done(): | |||
| 583 | + | return true | |||
| 584 | + | } | |||
| 585 | + | ||||
| 586 | + | running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID) | |||
| 587 | + | if err != nil || !running { | |||
| 588 | + | log.Error(). | |||
| 589 | + | Str("new_container", newContainer.ID). | |||
| 590 | + | Str("old_container", oldContainer.ID). | |||
| 591 | + | Msg("new container crashed during stabilization, rolling back to old") | |||
| 592 | + | ||||
| 593 | + | // Rollback: restore old container as tracked | |||
| 594 | + | s.mu.Lock() | |||
| 595 | + | s.containers[domainName] = oldContainer | |||
| 596 | + | s.mu.Unlock() | |||
| 597 | + | ||||
| 598 | + | // Re-invalidate proxy cache to point back to old | |||
| 599 | + | s.mu.RLock() | |||
| 600 | + | inv := s.cacheInvalidator | |||
| 601 | + | s.mu.RUnlock() | |||
| 602 | + | if inv != nil { | |||
| 603 | + | inv.InvalidateTarget(ctx, domainName) | |||
| 604 | + | } | |||
| 605 | + | ||||
| 606 | + | // Cleanup failed new container | |||
| 607 | + | _ = s.runtime.StopContainer(ctx, newContainer.ID) | |||
| 608 | + | _ = s.runtime.RemoveContainer(ctx, newContainer.ID, true) | |||
| 609 | + | ||||
| 610 | + | return false | |||
| 611 | + | } | |||
| 612 | + | ||||
| 613 | + | return true | |||
| 614 | + | } | |||
| 615 | + | ||||
| 552 | func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) { | 616 | func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) { | ||
| 553 | if !hasExisting { | 617 | if !hasExisting { | ||
| 554 | return | 618 | return | ||
M internal/usecase/container/service_test.go +230 -38
| @@ -415,8 +415,9 @@ func TestService_Deploy_ReplacesExistingContainer(t *testing.T) { | |||
| 415 | 415 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 416 | 416 | ||
| 417 | 417 | config := Config{ | |
| 418 | - | ReadinessDelay: time.Millisecond, // Minimal delay for tests | |
| 419 | - | DrainDelay: time.Millisecond, | |
| 418 | + | ReadinessDelay: time.Millisecond, // Minimal delay for tests | |
| 419 | + | DrainDelay: time.Millisecond, | |
| 420 | + | StabilizationDelay: time.Millisecond, | |
| 420 | 421 | } | |
| 421 | 422 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 422 | 423 | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| @@ -454,8 +455,8 @@ func TestService_Deploy_ReplacesExistingContainer(t *testing.T) { | |||
| 454 | 455 | })).Return(newContainer, nil) | |
| 455 | 456 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 456 | 457 | ||
| 457 | - | // Wait for ready | |
| 458 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 458 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 459 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 459 | 460 | ||
| 460 | 461 | // Inspect after ready | |
| 461 | 462 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| @@ -469,7 +470,7 @@ func TestService_Deploy_ReplacesExistingContainer(t *testing.T) { | |||
| 469 | 470 | // Synchronous cache invalidation before old container cleanup | |
| 470 | 471 | cacheInvalidator.EXPECT().InvalidateTarget(mock.Anything, "test.example.com").Return() | |
| 471 | 472 | ||
| 472 | - | // Now cleanup old container (after new one is ready + cache invalidated + drain delay) | |
| 473 | + | // Now cleanup old container (after new one is ready + cache invalidated + stabilization + drain delay) | |
| 473 | 474 | runtime.EXPECT().StopContainer(mock.Anything, "old-container").Return(nil) | |
| 474 | 475 | runtime.EXPECT().RemoveContainer(mock.Anything, "old-container", true).Return(nil) | |
| 475 | 476 | ||
| @@ -493,8 +494,9 @@ func TestService_Deploy_ResolvesExistingFromRuntime_WhenMemoryStale(t *testing.T | |||
| 493 | 494 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 494 | 495 | ||
| 495 | 496 | config := Config{ | |
| 496 | - | ReadinessDelay: time.Millisecond, | |
| 497 | - | DrainDelay: time.Millisecond, | |
| 497 | + | ReadinessDelay: time.Millisecond, | |
| 498 | + | DrainDelay: time.Millisecond, | |
| 499 | + | StabilizationDelay: time.Millisecond, | |
| 498 | 500 | } | |
| 499 | 501 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 500 | 502 | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| @@ -551,8 +553,8 @@ func TestService_Deploy_ResolvesExistingFromRuntime_WhenMemoryStale(t *testing.T | |||
| 551 | 553 | })).Return(newContainer, nil) | |
| 552 | 554 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 553 | 555 | ||
| 554 | - | // Wait for ready | |
| 555 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 556 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 557 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 556 | 558 | ||
| 557 | 559 | // Inspect after ready | |
| 558 | 560 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| @@ -592,8 +594,9 @@ func TestService_Deploy_SkipRedundantDeploy_GetImageIDError(t *testing.T) { | |||
| 592 | 594 | eventBus := mocks.NewMockEventPublisher(t) | |
| 593 | 595 | ||
| 594 | 596 | config := Config{ | |
| 595 | - | ReadinessDelay: time.Millisecond, | |
| 596 | - | DrainDelay: time.Millisecond, | |
| 597 | + | ReadinessDelay: time.Millisecond, | |
| 598 | + | DrainDelay: time.Millisecond, | |
| 599 | + | StabilizationDelay: time.Millisecond, | |
| 597 | 600 | } | |
| 598 | 601 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 599 | 602 | ctx := testContext() | |
| @@ -627,7 +630,8 @@ func TestService_Deploy_SkipRedundantDeploy_GetImageIDError(t *testing.T) { | |||
| 627 | 630 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |
| 628 | 631 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | |
| 629 | 632 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 630 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 633 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 634 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 631 | 635 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 632 | 636 | ID: "new-container", Status: "running", | |
| 633 | 637 | }, nil) | |
| @@ -1667,8 +1671,9 @@ func TestService_Deploy_OrphanCleanupSkipsTrackedContainer(t *testing.T) { | |||
| 1667 | 1671 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 1668 | 1672 | ||
| 1669 | 1673 | config := Config{ | |
| 1670 | - | ReadinessDelay: time.Millisecond, // Minimal delay for tests | |
| 1671 | - | DrainDelay: time.Millisecond, | |
| 1674 | + | ReadinessDelay: time.Millisecond, // Minimal delay for tests | |
| 1675 | + | DrainDelay: time.Millisecond, | |
| 1676 | + | StabilizationDelay: time.Millisecond, | |
| 1672 | 1677 | } | |
| 1673 | 1678 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 1674 | 1679 | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| @@ -1714,8 +1719,8 @@ func TestService_Deploy_OrphanCleanupSkipsTrackedContainer(t *testing.T) { | |||
| 1714 | 1719 | })).Return(newContainer, nil) | |
| 1715 | 1720 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 1716 | 1721 | ||
| 1717 | - | // Wait for ready | |
| 1718 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 1722 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 1723 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 1719 | 1724 | ||
| 1720 | 1725 | // Inspect after ready | |
| 1721 | 1726 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| @@ -1835,8 +1840,9 @@ func TestService_Deploy_OrphanCleanupRemovesStaleNewContainer(t *testing.T) { | |||
| 1835 | 1840 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 1836 | 1841 | ||
| 1837 | 1842 | config := Config{ | |
| 1838 | - | ReadinessDelay: time.Millisecond, | |
| 1839 | - | DrainDelay: time.Millisecond, | |
| 1843 | + | ReadinessDelay: time.Millisecond, | |
| 1844 | + | DrainDelay: time.Millisecond, | |
| 1845 | + | StabilizationDelay: time.Millisecond, | |
| 1840 | 1846 | } | |
| 1841 | 1847 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 1842 | 1848 | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| @@ -1880,7 +1886,8 @@ func TestService_Deploy_OrphanCleanupRemovesStaleNewContainer(t *testing.T) { | |||
| 1880 | 1886 | return cfg.Name == "gordon-test.example.com-new" | |
| 1881 | 1887 | })).Return(newContainer, nil) | |
| 1882 | 1888 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 1883 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 1889 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 1890 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 1884 | 1891 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 1885 | 1892 | ID: "new-container", | |
| 1886 | 1893 | Status: "running", | |
| @@ -1907,8 +1914,9 @@ func TestService_Deploy_TrackedTempContainerUsesAlternateTempName(t *testing.T) | |||
| 1907 | 1914 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 1908 | 1915 | ||
| 1909 | 1916 | config := Config{ | |
| 1910 | - | ReadinessDelay: time.Millisecond, | |
| 1911 | - | DrainDelay: time.Millisecond, | |
| 1917 | + | ReadinessDelay: time.Millisecond, | |
| 1918 | + | DrainDelay: time.Millisecond, | |
| 1919 | + | StabilizationDelay: time.Millisecond, | |
| 1912 | 1920 | } | |
| 1913 | 1921 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 1914 | 1922 | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| @@ -1945,7 +1953,8 @@ func TestService_Deploy_TrackedTempContainerUsesAlternateTempName(t *testing.T) | |||
| 1945 | 1953 | return cfg.Name == "gordon-test.example.com-next" | |
| 1946 | 1954 | })).Return(created, nil) | |
| 1947 | 1955 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 1948 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 1956 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 1957 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 1949 | 1958 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 1950 | 1959 | ID: "new-container", | |
| 1951 | 1960 | Status: "running", | |
| @@ -2096,8 +2105,9 @@ func TestService_Deploy_ConcurrentSameDomain(t *testing.T) { | |||
| 2096 | 2105 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 2097 | 2106 | ||
| 2098 | 2107 | config := Config{ | |
| 2099 | - | ReadinessDelay: time.Millisecond, | |
| 2100 | - | DrainDelay: time.Millisecond, | |
| 2108 | + | ReadinessDelay: time.Millisecond, | |
| 2109 | + | DrainDelay: time.Millisecond, | |
| 2110 | + | StabilizationDelay: time.Millisecond, | |
| 2101 | 2111 | } | |
| 2102 | 2112 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 2103 | 2113 | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| @@ -2142,8 +2152,8 @@ func TestService_Deploy_ConcurrentSameDomain(t *testing.T) { | |||
| 2142 | 2152 | }) | |
| 2143 | 2153 | ||
| 2144 | 2154 | runtime.EXPECT().StartContainer(mock.Anything, mock.AnythingOfType("string")).Return(nil).Times(2) | |
| 2145 | - | // IsContainerRunning is called 2x per deploy in waitForReady (initial check + after delay) | |
| 2146 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, mock.AnythingOfType("string")).Return(true, nil).Times(4) | |
| 2155 | + | // IsContainerRunning: 2x per deploy in waitForReady + 1x stabilization for the second deploy (which has existing) | |
| 2156 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, mock.AnythingOfType("string")).Return(true, nil).Times(5) | |
| 2147 | 2157 | ||
| 2148 | 2158 | runtime.EXPECT().InspectContainer(mock.Anything, mock.AnythingOfType("string")). | |
| 2149 | 2159 | RunAndReturn(func(_ context.Context, id string) (*domain.Container, error) { | |
| @@ -2235,8 +2245,9 @@ func TestService_Deploy_CacheInvalidationBeforeOldContainerStop(t *testing.T) { | |||
| 2235 | 2245 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 2236 | 2246 | ||
| 2237 | 2247 | config := Config{ | |
| 2238 | - | ReadinessDelay: time.Millisecond, | |
| 2239 | - | DrainDelay: time.Millisecond, | |
| 2248 | + | ReadinessDelay: time.Millisecond, | |
| 2249 | + | DrainDelay: time.Millisecond, | |
| 2250 | + | StabilizationDelay: time.Millisecond, | |
| 2240 | 2251 | } | |
| 2241 | 2252 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 2242 | 2253 | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| @@ -2269,7 +2280,8 @@ func TestService_Deploy_CacheInvalidationBeforeOldContainerStop(t *testing.T) { | |||
| 2269 | 2280 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |
| 2270 | 2281 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | |
| 2271 | 2282 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 2272 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 2283 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 2284 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 2273 | 2285 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 2274 | 2286 | ID: "new-container", Status: "running", | |
| 2275 | 2287 | }, nil) | |
| @@ -2314,8 +2326,9 @@ func TestService_Deploy_NilCacheInvalidator(t *testing.T) { | |||
| 2314 | 2326 | eventBus := mocks.NewMockEventPublisher(t) | |
| 2315 | 2327 | ||
| 2316 | 2328 | config := Config{ | |
| 2317 | - | ReadinessDelay: time.Millisecond, | |
| 2318 | - | DrainDelay: time.Millisecond, | |
| 2329 | + | ReadinessDelay: time.Millisecond, | |
| 2330 | + | DrainDelay: time.Millisecond, | |
| 2331 | + | StabilizationDelay: time.Millisecond, | |
| 2319 | 2332 | } | |
| 2320 | 2333 | // Intentionally NOT setting cache invalidator | |
| 2321 | 2334 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| @@ -2342,7 +2355,8 @@ func TestService_Deploy_NilCacheInvalidator(t *testing.T) { | |||
| 2342 | 2355 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |
| 2343 | 2356 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | |
| 2344 | 2357 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 2345 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 2358 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 2359 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 2346 | 2360 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 2347 | 2361 | ID: "new-container", Status: "running", | |
| 2348 | 2362 | }, nil) | |
| @@ -2471,8 +2485,9 @@ func TestService_Deploy_DoesNotSkipWhenImageIDDiffers(t *testing.T) { | |||
| 2471 | 2485 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 2472 | 2486 | ||
| 2473 | 2487 | config := Config{ | |
| 2474 | - | ReadinessDelay: time.Millisecond, | |
| 2475 | - | DrainDelay: time.Millisecond, | |
| 2488 | + | ReadinessDelay: time.Millisecond, | |
| 2489 | + | DrainDelay: time.Millisecond, | |
| 2490 | + | StabilizationDelay: time.Millisecond, | |
| 2476 | 2491 | } | |
| 2477 | 2492 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 2478 | 2493 | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| @@ -2507,7 +2522,8 @@ func TestService_Deploy_DoesNotSkipWhenImageIDDiffers(t *testing.T) { | |||
| 2507 | 2522 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |
| 2508 | 2523 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | |
| 2509 | 2524 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 2510 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 2525 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 2526 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 2511 | 2527 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 2512 | 2528 | ID: "new-container", Status: "running", | |
| 2513 | 2529 | }, nil) | |
| @@ -2572,8 +2588,9 @@ func TestService_Deploy_SkipRedundantDeploy_ContainerNotRunning(t *testing.T) { | |||
| 2572 | 2588 | eventBus := mocks.NewMockEventPublisher(t) | |
| 2573 | 2589 | ||
| 2574 | 2590 | config := Config{ | |
| 2575 | - | ReadinessDelay: time.Millisecond, | |
| 2576 | - | DrainDelay: time.Millisecond, | |
| 2591 | + | ReadinessDelay: time.Millisecond, | |
| 2592 | + | DrainDelay: time.Millisecond, | |
| 2593 | + | StabilizationDelay: time.Millisecond, | |
| 2577 | 2594 | } | |
| 2578 | 2595 | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 2579 | 2596 | ctx := testContext() | |
| @@ -2608,7 +2625,8 @@ func TestService_Deploy_SkipRedundantDeploy_ContainerNotRunning(t *testing.T) { | |||
| 2608 | 2625 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |
| 2609 | 2626 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | |
| 2610 | 2627 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 2611 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 2628 | + | // Wait for ready (2 calls) + stabilization check (1 call) | |
| 2629 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 2612 | 2630 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 2613 | 2631 | ID: "new-container", Status: "running", | |
| 2614 | 2632 | }, nil) | |
| @@ -2712,3 +2730,177 @@ func TestService_Deploy_OrphanCleanup_NeverKillsRunningCanonical(t *testing.T) { | |||
| 2712 | 2730 | assert.True(t, exists) | |
| 2713 | 2731 | assert.Equal(t, "new-container", tracked.ID) | |
| 2714 | 2732 | } | |
| 2733 | + | ||
| 2734 | + | // TestService_Deploy_RollbackOnPostSwitchCrash verifies that if the new container crashes | |
| 2735 | + | // during the post-switch stabilization window, traffic is automatically rolled back to the | |
| 2736 | + | // old container: the old container is restored as tracked, the proxy cache is re-invalidated, | |
| 2737 | + | // and the failed new container is cleaned up. | |
| 2738 | + | func TestService_Deploy_RollbackOnPostSwitchCrash(t *testing.T) { | |
| 2739 | + | runtime := mocks.NewMockContainerRuntime(t) | |
| 2740 | + | envLoader := mocks.NewMockEnvLoader(t) | |
| 2741 | + | eventBus := mocks.NewMockEventPublisher(t) | |
| 2742 | + | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 2743 | + | ||
| 2744 | + | config := Config{ | |
| 2745 | + | ReadinessDelay: time.Millisecond, | |
| 2746 | + | DrainDelay: time.Millisecond, | |
| 2747 | + | DrainDelayConfigured: true, | |
| 2748 | + | StabilizationDelay: time.Millisecond, | |
| 2749 | + | } | |
| 2750 | + | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 2751 | + | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| 2752 | + | ctx := testContext() | |
| 2753 | + | ||
| 2754 | + | // Pre-populate with existing container | |
| 2755 | + | existing := &domain.Container{ | |
| 2756 | + | ID: "old-container", | |
| 2757 | + | Name: "gordon-test.example.com", | |
| 2758 | + | Status: "running", | |
| 2759 | + | } | |
| 2760 | + | svc.containers["test.example.com"] = existing | |
| 2761 | + | ||
| 2762 | + | route := domain.Route{ | |
| 2763 | + | Domain: "test.example.com", | |
| 2764 | + | Image: "myapp:v2", | |
| 2765 | + | } | |
| 2766 | + | ||
| 2767 | + | // Cleanup orphans | |
| 2768 | + | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |
| 2769 | + | ||
| 2770 | + | // Image operations | |
| 2771 | + | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | |
| 2772 | + | runtime.EXPECT().PullImage(mock.Anything, "myapp:v2").Return(nil) | |
| 2773 | + | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:v2").Return([]int{8080}, nil) | |
| 2774 | + | ||
| 2775 | + | // Environment | |
| 2776 | + | envLoader.EXPECT().LoadEnv(mock.Anything, "test.example.com").Return([]string{}, nil) | |
| 2777 | + | runtime.EXPECT().InspectImageEnv(mock.Anything, "myapp:v2").Return([]string{}, nil) | |
| 2778 | + | ||
| 2779 | + | // Create new container | |
| 2780 | + | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |
| 2781 | + | runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool { | |
| 2782 | + | return cfg.Name == "gordon-test.example.com-new" | |
| 2783 | + | })).Return(newContainer, nil) | |
| 2784 | + | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 2785 | + | ||
| 2786 | + | // Readiness: new container is running during readiness checks (2 calls) | |
| 2787 | + | // Stabilization: new container has crashed (returns false) | |
| 2788 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 2789 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(false, nil).Once() | |
| 2790 | + | ||
| 2791 | + | // Inspect after ready | |
| 2792 | + | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 2793 | + | ID: "new-container", | |
| 2794 | + | Name: "gordon-test.example.com-new", | |
| 2795 | + | Status: "running", | |
| 2796 | + | }, nil) | |
| 2797 | + | ||
| 2798 | + | // Publish event (happens during activateDeployedContainer) | |
| 2799 | + | eventBus.EXPECT().Publish(domain.EventContainerDeployed, mock.AnythingOfType("*domain.ContainerEventPayload")).Return(nil) | |
| 2800 | + | ||
| 2801 | + | // InvalidateTarget called TWICE: once during activate, once during rollback | |
| 2802 | + | cacheInvalidator.EXPECT().InvalidateTarget(mock.Anything, "test.example.com").Return().Times(2) | |
| 2803 | + | ||
| 2804 | + | // Cleanup failed new container during rollback | |
| 2805 | + | runtime.EXPECT().StopContainer(mock.Anything, "new-container").Return(nil) | |
| 2806 | + | runtime.EXPECT().RemoveContainer(mock.Anything, "new-container", true).Return(nil) | |
| 2807 | + | ||
| 2808 | + | // StopContainer on old-container should NOT be called (old stays running) | |
| 2809 | + | // (no expectation for StopContainer("old-container")) | |
| 2810 | + | ||
| 2811 | + | result, err := svc.Deploy(ctx, route) | |
| 2812 | + | ||
| 2813 | + | // Deploy returns the old (existing) container after rollback | |
| 2814 | + | assert.NoError(t, err) | |
| 2815 | + | assert.NotNil(t, result) | |
| 2816 | + | assert.Equal(t, "old-container", result.ID, "should return old container after rollback") | |
| 2817 | + | ||
| 2818 | + | // Verify old container is restored as tracked | |
| 2819 | + | trackedRB, existsRB := svc.Get(ctx, "test.example.com") | |
| 2820 | + | assert.True(t, existsRB) | |
| 2821 | + | assert.Equal(t, "old-container", trackedRB.ID, "old container should be restored after rollback") | |
| 2822 | + | } | |
| 2823 | + | ||
| 2824 | + | // TestService_Deploy_StabilizationSuccess verifies that when the new container remains | |
| 2825 | + | // healthy during the stabilization window, deployment proceeds normally and the old | |
| 2826 | + | // container is finalized (stopped and removed). | |
| 2827 | + | func TestService_Deploy_StabilizationSuccess(t *testing.T) { | |
| 2828 | + | runtime := mocks.NewMockContainerRuntime(t) | |
| 2829 | + | envLoader := mocks.NewMockEnvLoader(t) | |
| 2830 | + | eventBus := mocks.NewMockEventPublisher(t) | |
| 2831 | + | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 2832 | + | ||
| 2833 | + | config := Config{ | |
| 2834 | + | ReadinessDelay: time.Millisecond, | |
| 2835 | + | DrainDelay: time.Millisecond, | |
| 2836 | + | DrainDelayConfigured: true, | |
| 2837 | + | StabilizationDelay: time.Millisecond, | |
| 2838 | + | } | |
| 2839 | + | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 2840 | + | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| 2841 | + | ctx := testContext() | |
| 2842 | + | ||
| 2843 | + | // Pre-populate with existing container | |
| 2844 | + | existing := &domain.Container{ | |
| 2845 | + | ID: "old-container", | |
| 2846 | + | Name: "gordon-test.example.com", | |
| 2847 | + | Status: "running", | |
| 2848 | + | } | |
| 2849 | + | svc.containers["test.example.com"] = existing | |
| 2850 | + | ||
| 2851 | + | route := domain.Route{ | |
| 2852 | + | Domain: "test.example.com", | |
| 2853 | + | Image: "myapp:v2", | |
| 2854 | + | } | |
| 2855 | + | ||
| 2856 | + | // Cleanup orphans | |
| 2857 | + | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |
| 2858 | + | ||
| 2859 | + | // Image operations | |
| 2860 | + | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | |
| 2861 | + | runtime.EXPECT().PullImage(mock.Anything, "myapp:v2").Return(nil) | |
| 2862 | + | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:v2").Return([]int{8080}, nil) | |
| 2863 | + | ||
| 2864 | + | // Environment | |
| 2865 | + | envLoader.EXPECT().LoadEnv(mock.Anything, "test.example.com").Return([]string{}, nil) | |
| 2866 | + | runtime.EXPECT().InspectImageEnv(mock.Anything, "myapp:v2").Return([]string{}, nil) | |
| 2867 | + | ||
| 2868 | + | // Create new container | |
| 2869 | + | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |
| 2870 | + | runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool { | |
| 2871 | + | return cfg.Name == "gordon-test.example.com-new" | |
| 2872 | + | })).Return(newContainer, nil) | |
| 2873 | + | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 2874 | + | ||
| 2875 | + | // Readiness (2 calls) + stabilization check (1 call) — all return running | |
| 2876 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |
| 2877 | + | ||
| 2878 | + | // Inspect after ready | |
| 2879 | + | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 2880 | + | ID: "new-container", | |
| 2881 | + | Name: "gordon-test.example.com-new", | |
| 2882 | + | Status: "running", | |
| 2883 | + | }, nil) | |
| 2884 | + | ||
| 2885 | + | // Publish event | |
| 2886 | + | eventBus.EXPECT().Publish(domain.EventContainerDeployed, mock.AnythingOfType("*domain.ContainerEventPayload")).Return(nil) | |
| 2887 | + | ||
| 2888 | + | // Cache invalidation (once, during activate) | |
| 2889 | + | cacheInvalidator.EXPECT().InvalidateTarget(mock.Anything, "test.example.com").Return() | |
| 2890 | + | ||
| 2891 | + | // Old container finalized normally | |
| 2892 | + | runtime.EXPECT().StopContainer(mock.Anything, "old-container").Return(nil) | |
| 2893 | + | runtime.EXPECT().RemoveContainer(mock.Anything, "old-container", true).Return(nil) | |
| 2894 | + | runtime.EXPECT().RenameContainer(mock.Anything, "new-container", "gordon-test.example.com").Return(nil) | |
| 2895 | + | ||
| 2896 | + | result, err := svc.Deploy(ctx, route) | |
| 2897 | + | ||
| 2898 | + | assert.NoError(t, err) | |
| 2899 | + | assert.NotNil(t, result) | |
| 2900 | + | assert.Equal(t, "new-container", result.ID, "should return new container on successful stabilization") | |
| 2901 | + | ||
| 2902 | + | // Verify new container is tracked | |
| 2903 | + | trackedSS, existsSS := svc.Get(ctx, "test.example.com") | |
| 2904 | + | assert.True(t, existsSS) | |
| 2905 | + | assert.Equal(t, "new-container", trackedSS.ID) | |
| 2906 | + | } | |
| @@ -415,8 +415,9 @@ func TestService_Deploy_ReplacesExistingContainer(t *testing.T) { | |||||
| 415 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | 415 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | ||
| 416 | 416 | ||||
| 417 | config := Config{ | 417 | config := Config{ | ||
| 418 | - | ReadinessDelay: time.Millisecond, // Minimal delay for tests | 418 | + | ReadinessDelay: time.Millisecond, // Minimal delay for tests |
| 419 | - | DrainDelay: time.Millisecond, | 419 | + | DrainDelay: time.Millisecond, |
| 420 | + | StabilizationDelay: time.Millisecond, | |||
| 420 | } | 421 | } | ||
| 421 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 422 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 422 | svc.SetProxyCacheInvalidator(cacheInvalidator) | 423 | svc.SetProxyCacheInvalidator(cacheInvalidator) | ||
| @@ -454,8 +455,8 @@ func TestService_Deploy_ReplacesExistingContainer(t *testing.T) { | |||||
| 454 | })).Return(newContainer, nil) | 455 | })).Return(newContainer, nil) | ||
| 455 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 456 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 456 | 457 | ||||
| 457 | - | // Wait for ready | 458 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 458 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 459 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) |
| 459 | 460 | ||||
| 460 | // Inspect after ready | 461 | // Inspect after ready | ||
| 461 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 462 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| @@ -469,7 +470,7 @@ func TestService_Deploy_ReplacesExistingContainer(t *testing.T) { | |||||
| 469 | // Synchronous cache invalidation before old container cleanup | 470 | // Synchronous cache invalidation before old container cleanup | ||
| 470 | cacheInvalidator.EXPECT().InvalidateTarget(mock.Anything, "test.example.com").Return() | 471 | cacheInvalidator.EXPECT().InvalidateTarget(mock.Anything, "test.example.com").Return() | ||
| 471 | 472 | ||||
| 472 | - | // Now cleanup old container (after new one is ready + cache invalidated + drain delay) | 473 | + | // Now cleanup old container (after new one is ready + cache invalidated + stabilization + drain delay) |
| 473 | runtime.EXPECT().StopContainer(mock.Anything, "old-container").Return(nil) | 474 | runtime.EXPECT().StopContainer(mock.Anything, "old-container").Return(nil) | ||
| 474 | runtime.EXPECT().RemoveContainer(mock.Anything, "old-container", true).Return(nil) | 475 | runtime.EXPECT().RemoveContainer(mock.Anything, "old-container", true).Return(nil) | ||
| 475 | 476 | ||||
| @@ -493,8 +494,9 @@ func TestService_Deploy_ResolvesExistingFromRuntime_WhenMemoryStale(t *testing.T | |||||
| 493 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | 494 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | ||
| 494 | 495 | ||||
| 495 | config := Config{ | 496 | config := Config{ | ||
| 496 | - | ReadinessDelay: time.Millisecond, | 497 | + | ReadinessDelay: time.Millisecond, |
| 497 | - | DrainDelay: time.Millisecond, | 498 | + | DrainDelay: time.Millisecond, |
| 499 | + | StabilizationDelay: time.Millisecond, | |||
| 498 | } | 500 | } | ||
| 499 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 501 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 500 | svc.SetProxyCacheInvalidator(cacheInvalidator) | 502 | svc.SetProxyCacheInvalidator(cacheInvalidator) | ||
| @@ -551,8 +553,8 @@ func TestService_Deploy_ResolvesExistingFromRuntime_WhenMemoryStale(t *testing.T | |||||
| 551 | })).Return(newContainer, nil) | 553 | })).Return(newContainer, nil) | ||
| 552 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 554 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 553 | 555 | ||||
| 554 | - | // Wait for ready | 556 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 555 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 557 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) |
| 556 | 558 | ||||
| 557 | // Inspect after ready | 559 | // Inspect after ready | ||
| 558 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 560 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| @@ -592,8 +594,9 @@ func TestService_Deploy_SkipRedundantDeploy_GetImageIDError(t *testing.T) { | |||||
| 592 | eventBus := mocks.NewMockEventPublisher(t) | 594 | eventBus := mocks.NewMockEventPublisher(t) | ||
| 593 | 595 | ||||
| 594 | config := Config{ | 596 | config := Config{ | ||
| 595 | - | ReadinessDelay: time.Millisecond, | 597 | + | ReadinessDelay: time.Millisecond, |
| 596 | - | DrainDelay: time.Millisecond, | 598 | + | DrainDelay: time.Millisecond, |
| 599 | + | StabilizationDelay: time.Millisecond, | |||
| 597 | } | 600 | } | ||
| 598 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 601 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 599 | ctx := testContext() | 602 | ctx := testContext() | ||
| @@ -627,7 +630,8 @@ func TestService_Deploy_SkipRedundantDeploy_GetImageIDError(t *testing.T) { | |||||
| 627 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | 630 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | ||
| 628 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | 631 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | ||
| 629 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 632 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 630 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 633 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 634 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |||
| 631 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 635 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| 632 | ID: "new-container", Status: "running", | 636 | ID: "new-container", Status: "running", | ||
| 633 | }, nil) | 637 | }, nil) | ||
| @@ -1667,8 +1671,9 @@ func TestService_Deploy_OrphanCleanupSkipsTrackedContainer(t *testing.T) { | |||||
| 1667 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | 1671 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | ||
| 1668 | 1672 | ||||
| 1669 | config := Config{ | 1673 | config := Config{ | ||
| 1670 | - | ReadinessDelay: time.Millisecond, // Minimal delay for tests | 1674 | + | ReadinessDelay: time.Millisecond, // Minimal delay for tests |
| 1671 | - | DrainDelay: time.Millisecond, | 1675 | + | DrainDelay: time.Millisecond, |
| 1676 | + | StabilizationDelay: time.Millisecond, | |||
| 1672 | } | 1677 | } | ||
| 1673 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 1678 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 1674 | svc.SetProxyCacheInvalidator(cacheInvalidator) | 1679 | svc.SetProxyCacheInvalidator(cacheInvalidator) | ||
| @@ -1714,8 +1719,8 @@ func TestService_Deploy_OrphanCleanupSkipsTrackedContainer(t *testing.T) { | |||||
| 1714 | })).Return(newContainer, nil) | 1719 | })).Return(newContainer, nil) | ||
| 1715 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 1720 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 1716 | 1721 | ||||
| 1717 | - | // Wait for ready | 1722 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 1718 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 1723 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) |
| 1719 | 1724 | ||||
| 1720 | // Inspect after ready | 1725 | // Inspect after ready | ||
| 1721 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 1726 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| @@ -1835,8 +1840,9 @@ func TestService_Deploy_OrphanCleanupRemovesStaleNewContainer(t *testing.T) { | |||||
| 1835 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | 1840 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | ||
| 1836 | 1841 | ||||
| 1837 | config := Config{ | 1842 | config := Config{ | ||
| 1838 | - | ReadinessDelay: time.Millisecond, | 1843 | + | ReadinessDelay: time.Millisecond, |
| 1839 | - | DrainDelay: time.Millisecond, | 1844 | + | DrainDelay: time.Millisecond, |
| 1845 | + | StabilizationDelay: time.Millisecond, | |||
| 1840 | } | 1846 | } | ||
| 1841 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 1847 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 1842 | svc.SetProxyCacheInvalidator(cacheInvalidator) | 1848 | svc.SetProxyCacheInvalidator(cacheInvalidator) | ||
| @@ -1880,7 +1886,8 @@ func TestService_Deploy_OrphanCleanupRemovesStaleNewContainer(t *testing.T) { | |||||
| 1880 | return cfg.Name == "gordon-test.example.com-new" | 1886 | return cfg.Name == "gordon-test.example.com-new" | ||
| 1881 | })).Return(newContainer, nil) | 1887 | })).Return(newContainer, nil) | ||
| 1882 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 1888 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 1883 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 1889 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 1890 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |||
| 1884 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 1891 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| 1885 | ID: "new-container", | 1892 | ID: "new-container", | ||
| 1886 | Status: "running", | 1893 | Status: "running", | ||
| @@ -1907,8 +1914,9 @@ func TestService_Deploy_TrackedTempContainerUsesAlternateTempName(t *testing.T) | |||||
| 1907 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | 1914 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | ||
| 1908 | 1915 | ||||
| 1909 | config := Config{ | 1916 | config := Config{ | ||
| 1910 | - | ReadinessDelay: time.Millisecond, | 1917 | + | ReadinessDelay: time.Millisecond, |
| 1911 | - | DrainDelay: time.Millisecond, | 1918 | + | DrainDelay: time.Millisecond, |
| 1919 | + | StabilizationDelay: time.Millisecond, | |||
| 1912 | } | 1920 | } | ||
| 1913 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 1921 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 1914 | svc.SetProxyCacheInvalidator(cacheInvalidator) | 1922 | svc.SetProxyCacheInvalidator(cacheInvalidator) | ||
| @@ -1945,7 +1953,8 @@ func TestService_Deploy_TrackedTempContainerUsesAlternateTempName(t *testing.T) | |||||
| 1945 | return cfg.Name == "gordon-test.example.com-next" | 1953 | return cfg.Name == "gordon-test.example.com-next" | ||
| 1946 | })).Return(created, nil) | 1954 | })).Return(created, nil) | ||
| 1947 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 1955 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 1948 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 1956 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 1957 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |||
| 1949 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 1958 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| 1950 | ID: "new-container", | 1959 | ID: "new-container", | ||
| 1951 | Status: "running", | 1960 | Status: "running", | ||
| @@ -2096,8 +2105,9 @@ func TestService_Deploy_ConcurrentSameDomain(t *testing.T) { | |||||
| 2096 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | 2105 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | ||
| 2097 | 2106 | ||||
| 2098 | config := Config{ | 2107 | config := Config{ | ||
| 2099 | - | ReadinessDelay: time.Millisecond, | 2108 | + | ReadinessDelay: time.Millisecond, |
| 2100 | - | DrainDelay: time.Millisecond, | 2109 | + | DrainDelay: time.Millisecond, |
| 2110 | + | StabilizationDelay: time.Millisecond, | |||
| 2101 | } | 2111 | } | ||
| 2102 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 2112 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 2103 | svc.SetProxyCacheInvalidator(cacheInvalidator) | 2113 | svc.SetProxyCacheInvalidator(cacheInvalidator) | ||
| @@ -2142,8 +2152,8 @@ func TestService_Deploy_ConcurrentSameDomain(t *testing.T) { | |||||
| 2142 | }) | 2152 | }) | ||
| 2143 | 2153 | ||||
| 2144 | runtime.EXPECT().StartContainer(mock.Anything, mock.AnythingOfType("string")).Return(nil).Times(2) | 2154 | runtime.EXPECT().StartContainer(mock.Anything, mock.AnythingOfType("string")).Return(nil).Times(2) | ||
| 2145 | - | // IsContainerRunning is called 2x per deploy in waitForReady (initial check + after delay) | 2155 | + | // IsContainerRunning: 2x per deploy in waitForReady + 1x stabilization for the second deploy (which has existing) |
| 2146 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, mock.AnythingOfType("string")).Return(true, nil).Times(4) | 2156 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, mock.AnythingOfType("string")).Return(true, nil).Times(5) |
| 2147 | 2157 | ||||
| 2148 | runtime.EXPECT().InspectContainer(mock.Anything, mock.AnythingOfType("string")). | 2158 | runtime.EXPECT().InspectContainer(mock.Anything, mock.AnythingOfType("string")). | ||
| 2149 | RunAndReturn(func(_ context.Context, id string) (*domain.Container, error) { | 2159 | RunAndReturn(func(_ context.Context, id string) (*domain.Container, error) { | ||
| @@ -2235,8 +2245,9 @@ func TestService_Deploy_CacheInvalidationBeforeOldContainerStop(t *testing.T) { | |||||
| 2235 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | 2245 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | ||
| 2236 | 2246 | ||||
| 2237 | config := Config{ | 2247 | config := Config{ | ||
| 2238 | - | ReadinessDelay: time.Millisecond, | 2248 | + | ReadinessDelay: time.Millisecond, |
| 2239 | - | DrainDelay: time.Millisecond, | 2249 | + | DrainDelay: time.Millisecond, |
| 2250 | + | StabilizationDelay: time.Millisecond, | |||
| 2240 | } | 2251 | } | ||
| 2241 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 2252 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 2242 | svc.SetProxyCacheInvalidator(cacheInvalidator) | 2253 | svc.SetProxyCacheInvalidator(cacheInvalidator) | ||
| @@ -2269,7 +2280,8 @@ func TestService_Deploy_CacheInvalidationBeforeOldContainerStop(t *testing.T) { | |||||
| 2269 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | 2280 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | ||
| 2270 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | 2281 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | ||
| 2271 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 2282 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 2272 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 2283 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 2284 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |||
| 2273 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 2285 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| 2274 | ID: "new-container", Status: "running", | 2286 | ID: "new-container", Status: "running", | ||
| 2275 | }, nil) | 2287 | }, nil) | ||
| @@ -2314,8 +2326,9 @@ func TestService_Deploy_NilCacheInvalidator(t *testing.T) { | |||||
| 2314 | eventBus := mocks.NewMockEventPublisher(t) | 2326 | eventBus := mocks.NewMockEventPublisher(t) | ||
| 2315 | 2327 | ||||
| 2316 | config := Config{ | 2328 | config := Config{ | ||
| 2317 | - | ReadinessDelay: time.Millisecond, | 2329 | + | ReadinessDelay: time.Millisecond, |
| 2318 | - | DrainDelay: time.Millisecond, | 2330 | + | DrainDelay: time.Millisecond, |
| 2331 | + | StabilizationDelay: time.Millisecond, | |||
| 2319 | } | 2332 | } | ||
| 2320 | // Intentionally NOT setting cache invalidator | 2333 | // Intentionally NOT setting cache invalidator | ||
| 2321 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 2334 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| @@ -2342,7 +2355,8 @@ func TestService_Deploy_NilCacheInvalidator(t *testing.T) { | |||||
| 2342 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | 2355 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | ||
| 2343 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | 2356 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | ||
| 2344 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 2357 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 2345 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 2358 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 2359 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |||
| 2346 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 2360 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| 2347 | ID: "new-container", Status: "running", | 2361 | ID: "new-container", Status: "running", | ||
| 2348 | }, nil) | 2362 | }, nil) | ||
| @@ -2471,8 +2485,9 @@ func TestService_Deploy_DoesNotSkipWhenImageIDDiffers(t *testing.T) { | |||||
| 2471 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | 2485 | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | ||
| 2472 | 2486 | ||||
| 2473 | config := Config{ | 2487 | config := Config{ | ||
| 2474 | - | ReadinessDelay: time.Millisecond, | 2488 | + | ReadinessDelay: time.Millisecond, |
| 2475 | - | DrainDelay: time.Millisecond, | 2489 | + | DrainDelay: time.Millisecond, |
| 2490 | + | StabilizationDelay: time.Millisecond, | |||
| 2476 | } | 2491 | } | ||
| 2477 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 2492 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 2478 | svc.SetProxyCacheInvalidator(cacheInvalidator) | 2493 | svc.SetProxyCacheInvalidator(cacheInvalidator) | ||
| @@ -2507,7 +2522,8 @@ func TestService_Deploy_DoesNotSkipWhenImageIDDiffers(t *testing.T) { | |||||
| 2507 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | 2522 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | ||
| 2508 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | 2523 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | ||
| 2509 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 2524 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 2510 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 2525 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 2526 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |||
| 2511 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 2527 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| 2512 | ID: "new-container", Status: "running", | 2528 | ID: "new-container", Status: "running", | ||
| 2513 | }, nil) | 2529 | }, nil) | ||
| @@ -2572,8 +2588,9 @@ func TestService_Deploy_SkipRedundantDeploy_ContainerNotRunning(t *testing.T) { | |||||
| 2572 | eventBus := mocks.NewMockEventPublisher(t) | 2588 | eventBus := mocks.NewMockEventPublisher(t) | ||
| 2573 | 2589 | ||||
| 2574 | config := Config{ | 2590 | config := Config{ | ||
| 2575 | - | ReadinessDelay: time.Millisecond, | 2591 | + | ReadinessDelay: time.Millisecond, |
| 2576 | - | DrainDelay: time.Millisecond, | 2592 | + | DrainDelay: time.Millisecond, |
| 2593 | + | StabilizationDelay: time.Millisecond, | |||
| 2577 | } | 2594 | } | ||
| 2578 | svc := NewService(runtime, envLoader, eventBus, nil, config) | 2595 | svc := NewService(runtime, envLoader, eventBus, nil, config) | ||
| 2579 | ctx := testContext() | 2596 | ctx := testContext() | ||
| @@ -2608,7 +2625,8 @@ func TestService_Deploy_SkipRedundantDeploy_ContainerNotRunning(t *testing.T) { | |||||
| 2608 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | 2625 | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | ||
| 2609 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | 2626 | runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(newContainer, nil) | ||
| 2610 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | 2627 | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | ||
| 2611 | - | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | 2628 | + | // Wait for ready (2 calls) + stabilization check (1 call) |
| 2629 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |||
| 2612 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | 2630 | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | ||
| 2613 | ID: "new-container", Status: "running", | 2631 | ID: "new-container", Status: "running", | ||
| 2614 | }, nil) | 2632 | }, nil) | ||
| @@ -2712,3 +2730,177 @@ func TestService_Deploy_OrphanCleanup_NeverKillsRunningCanonical(t *testing.T) { | |||||
| 2712 | assert.True(t, exists) | 2730 | assert.True(t, exists) | ||
| 2713 | assert.Equal(t, "new-container", tracked.ID) | 2731 | assert.Equal(t, "new-container", tracked.ID) | ||
| 2714 | } | 2732 | } | ||
| 2733 | + | ||||
| 2734 | + | // TestService_Deploy_RollbackOnPostSwitchCrash verifies that if the new container crashes | |||
| 2735 | + | // during the post-switch stabilization window, traffic is automatically rolled back to the | |||
| 2736 | + | // old container: the old container is restored as tracked, the proxy cache is re-invalidated, | |||
| 2737 | + | // and the failed new container is cleaned up. | |||
| 2738 | + | func TestService_Deploy_RollbackOnPostSwitchCrash(t *testing.T) { | |||
| 2739 | + | runtime := mocks.NewMockContainerRuntime(t) | |||
| 2740 | + | envLoader := mocks.NewMockEnvLoader(t) | |||
| 2741 | + | eventBus := mocks.NewMockEventPublisher(t) | |||
| 2742 | + | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |||
| 2743 | + | ||||
| 2744 | + | config := Config{ | |||
| 2745 | + | ReadinessDelay: time.Millisecond, | |||
| 2746 | + | DrainDelay: time.Millisecond, | |||
| 2747 | + | DrainDelayConfigured: true, | |||
| 2748 | + | StabilizationDelay: time.Millisecond, | |||
| 2749 | + | } | |||
| 2750 | + | svc := NewService(runtime, envLoader, eventBus, nil, config) | |||
| 2751 | + | svc.SetProxyCacheInvalidator(cacheInvalidator) | |||
| 2752 | + | ctx := testContext() | |||
| 2753 | + | ||||
| 2754 | + | // Pre-populate with existing container | |||
| 2755 | + | existing := &domain.Container{ | |||
| 2756 | + | ID: "old-container", | |||
| 2757 | + | Name: "gordon-test.example.com", | |||
| 2758 | + | Status: "running", | |||
| 2759 | + | } | |||
| 2760 | + | svc.containers["test.example.com"] = existing | |||
| 2761 | + | ||||
| 2762 | + | route := domain.Route{ | |||
| 2763 | + | Domain: "test.example.com", | |||
| 2764 | + | Image: "myapp:v2", | |||
| 2765 | + | } | |||
| 2766 | + | ||||
| 2767 | + | // Cleanup orphans | |||
| 2768 | + | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |||
| 2769 | + | ||||
| 2770 | + | // Image operations | |||
| 2771 | + | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | |||
| 2772 | + | runtime.EXPECT().PullImage(mock.Anything, "myapp:v2").Return(nil) | |||
| 2773 | + | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:v2").Return([]int{8080}, nil) | |||
| 2774 | + | ||||
| 2775 | + | // Environment | |||
| 2776 | + | envLoader.EXPECT().LoadEnv(mock.Anything, "test.example.com").Return([]string{}, nil) | |||
| 2777 | + | runtime.EXPECT().InspectImageEnv(mock.Anything, "myapp:v2").Return([]string{}, nil) | |||
| 2778 | + | ||||
| 2779 | + | // Create new container | |||
| 2780 | + | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |||
| 2781 | + | runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool { | |||
| 2782 | + | return cfg.Name == "gordon-test.example.com-new" | |||
| 2783 | + | })).Return(newContainer, nil) | |||
| 2784 | + | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |||
| 2785 | + | ||||
| 2786 | + | // Readiness: new container is running during readiness checks (2 calls) | |||
| 2787 | + | // Stabilization: new container has crashed (returns false) | |||
| 2788 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |||
| 2789 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(false, nil).Once() | |||
| 2790 | + | ||||
| 2791 | + | // Inspect after ready | |||
| 2792 | + | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |||
| 2793 | + | ID: "new-container", | |||
| 2794 | + | Name: "gordon-test.example.com-new", | |||
| 2795 | + | Status: "running", | |||
| 2796 | + | }, nil) | |||
| 2797 | + | ||||
| 2798 | + | // Publish event (happens during activateDeployedContainer) | |||
| 2799 | + | eventBus.EXPECT().Publish(domain.EventContainerDeployed, mock.AnythingOfType("*domain.ContainerEventPayload")).Return(nil) | |||
| 2800 | + | ||||
| 2801 | + | // InvalidateTarget called TWICE: once during activate, once during rollback | |||
| 2802 | + | cacheInvalidator.EXPECT().InvalidateTarget(mock.Anything, "test.example.com").Return().Times(2) | |||
| 2803 | + | ||||
| 2804 | + | // Cleanup failed new container during rollback | |||
| 2805 | + | runtime.EXPECT().StopContainer(mock.Anything, "new-container").Return(nil) | |||
| 2806 | + | runtime.EXPECT().RemoveContainer(mock.Anything, "new-container", true).Return(nil) | |||
| 2807 | + | ||||
| 2808 | + | // StopContainer on old-container should NOT be called (old stays running) | |||
| 2809 | + | // (no expectation for StopContainer("old-container")) | |||
| 2810 | + | ||||
| 2811 | + | result, err := svc.Deploy(ctx, route) | |||
| 2812 | + | ||||
| 2813 | + | // Deploy returns the old (existing) container after rollback | |||
| 2814 | + | assert.NoError(t, err) | |||
| 2815 | + | assert.NotNil(t, result) | |||
| 2816 | + | assert.Equal(t, "old-container", result.ID, "should return old container after rollback") | |||
| 2817 | + | ||||
| 2818 | + | // Verify old container is restored as tracked | |||
| 2819 | + | trackedRB, existsRB := svc.Get(ctx, "test.example.com") | |||
| 2820 | + | assert.True(t, existsRB) | |||
| 2821 | + | assert.Equal(t, "old-container", trackedRB.ID, "old container should be restored after rollback") | |||
| 2822 | + | } | |||
| 2823 | + | ||||
| 2824 | + | // TestService_Deploy_StabilizationSuccess verifies that when the new container remains | |||
| 2825 | + | // healthy during the stabilization window, deployment proceeds normally and the old | |||
| 2826 | + | // container is finalized (stopped and removed). | |||
| 2827 | + | func TestService_Deploy_StabilizationSuccess(t *testing.T) { | |||
| 2828 | + | runtime := mocks.NewMockContainerRuntime(t) | |||
| 2829 | + | envLoader := mocks.NewMockEnvLoader(t) | |||
| 2830 | + | eventBus := mocks.NewMockEventPublisher(t) | |||
| 2831 | + | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |||
| 2832 | + | ||||
| 2833 | + | config := Config{ | |||
| 2834 | + | ReadinessDelay: time.Millisecond, | |||
| 2835 | + | DrainDelay: time.Millisecond, | |||
| 2836 | + | DrainDelayConfigured: true, | |||
| 2837 | + | StabilizationDelay: time.Millisecond, | |||
| 2838 | + | } | |||
| 2839 | + | svc := NewService(runtime, envLoader, eventBus, nil, config) | |||
| 2840 | + | svc.SetProxyCacheInvalidator(cacheInvalidator) | |||
| 2841 | + | ctx := testContext() | |||
| 2842 | + | ||||
| 2843 | + | // Pre-populate with existing container | |||
| 2844 | + | existing := &domain.Container{ | |||
| 2845 | + | ID: "old-container", | |||
| 2846 | + | Name: "gordon-test.example.com", | |||
| 2847 | + | Status: "running", | |||
| 2848 | + | } | |||
| 2849 | + | svc.containers["test.example.com"] = existing | |||
| 2850 | + | ||||
| 2851 | + | route := domain.Route{ | |||
| 2852 | + | Domain: "test.example.com", | |||
| 2853 | + | Image: "myapp:v2", | |||
| 2854 | + | } | |||
| 2855 | + | ||||
| 2856 | + | // Cleanup orphans | |||
| 2857 | + | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |||
| 2858 | + | ||||
| 2859 | + | // Image operations | |||
| 2860 | + | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | |||
| 2861 | + | runtime.EXPECT().PullImage(mock.Anything, "myapp:v2").Return(nil) | |||
| 2862 | + | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:v2").Return([]int{8080}, nil) | |||
| 2863 | + | ||||
| 2864 | + | // Environment | |||
| 2865 | + | envLoader.EXPECT().LoadEnv(mock.Anything, "test.example.com").Return([]string{}, nil) | |||
| 2866 | + | runtime.EXPECT().InspectImageEnv(mock.Anything, "myapp:v2").Return([]string{}, nil) | |||
| 2867 | + | ||||
| 2868 | + | // Create new container | |||
| 2869 | + | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |||
| 2870 | + | runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool { | |||
| 2871 | + | return cfg.Name == "gordon-test.example.com-new" | |||
| 2872 | + | })).Return(newContainer, nil) | |||
| 2873 | + | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |||
| 2874 | + | ||||
| 2875 | + | // Readiness (2 calls) + stabilization check (1 call) — all return running | |||
| 2876 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(3) | |||
| 2877 | + | ||||
| 2878 | + | // Inspect after ready | |||
| 2879 | + | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |||
| 2880 | + | ID: "new-container", | |||
| 2881 | + | Name: "gordon-test.example.com-new", | |||
| 2882 | + | Status: "running", | |||
| 2883 | + | }, nil) | |||
| 2884 | + | ||||
| 2885 | + | // Publish event | |||
| 2886 | + | eventBus.EXPECT().Publish(domain.EventContainerDeployed, mock.AnythingOfType("*domain.ContainerEventPayload")).Return(nil) | |||
| 2887 | + | ||||
| 2888 | + | // Cache invalidation (once, during activate) | |||
| 2889 | + | cacheInvalidator.EXPECT().InvalidateTarget(mock.Anything, "test.example.com").Return() | |||
| 2890 | + | ||||
| 2891 | + | // Old container finalized normally | |||
| 2892 | + | runtime.EXPECT().StopContainer(mock.Anything, "old-container").Return(nil) | |||
| 2893 | + | runtime.EXPECT().RemoveContainer(mock.Anything, "old-container", true).Return(nil) | |||
| 2894 | + | runtime.EXPECT().RenameContainer(mock.Anything, "new-container", "gordon-test.example.com").Return(nil) | |||
| 2895 | + | ||||
| 2896 | + | result, err := svc.Deploy(ctx, route) | |||
| 2897 | + | ||||
| 2898 | + | assert.NoError(t, err) | |||
| 2899 | + | assert.NotNil(t, result) | |||
| 2900 | + | assert.Equal(t, "new-container", result.ID, "should return new container on successful stabilization") | |||
| 2901 | + | ||||
| 2902 | + | // Verify new container is tracked | |||
| 2903 | + | trackedSS, existsSS := svc.Get(ctx, "test.example.com") | |||
| 2904 | + | assert.True(t, existsSS) | |||
| 2905 | + | assert.Equal(t, "new-container", trackedSS.ID) | |||
| 2906 | + | } | |||