fix: resolve existing container from runtime when memory is stale
Deploy now queries the runtime for a running container with matching labels when in-memory state is empty. This prevents orphan cleanup from killing the active container after Gordon restarts.
- commit
- 3efe7b911e5add9f3f0d16120163d2e4f8e2da43
- author
- brice · · 2026-03-02 16:15 UTC
- parent
- b03b115
- signature
- unsigned
2 files · +189 -4
M internal/usecase/container/service.go +44 -1
| @@ -276,7 +276,7 @@ func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Conta | |||
| 276 | 276 | } | |
| 277 | 277 | }() | |
| 278 | 278 | ||
| 279 | - | existing, hasExisting := s.getTrackedContainer(route.Domain) | |
| 279 | + | existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain) | |
| 280 | 280 | ||
| 281 | 281 | resources, err := s.prepareDeployResources(ctx, route, existing) | |
| 282 | 282 | if err != nil { | |
| @@ -388,6 +388,49 @@ func (s *Service) getTrackedContainer(domainName string) (*domain.Container, boo | |||
| 388 | 388 | return container, ok | |
| 389 | 389 | } | |
| 390 | 390 | ||
| 391 | + | // resolveExistingContainer returns the currently running container for a domain. | |
| 392 | + | // It first checks the in-memory map (fast path). If not found, it queries the | |
| 393 | + | // runtime for a running container with matching name and managed label. This | |
| 394 | + | // handles cases where Gordon restarted and in-memory state is stale. | |
| 395 | + | func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) { | |
| 396 | + | // Fast path: check in-memory state | |
| 397 | + | s.mu.RLock() | |
| 398 | + | container, ok := s.containers[domainName] | |
| 399 | + | s.mu.RUnlock() | |
| 400 | + | if ok { | |
| 401 | + | return container, true | |
| 402 | + | } | |
| 403 | + | ||
| 404 | + | // Slow path: query runtime for running containers | |
| 405 | + | log := zerowrap.FromCtx(ctx) | |
| 406 | + | expectedName := fmt.Sprintf("gordon-%s", domainName) | |
| 407 | + | ||
| 408 | + | running, err := s.runtime.ListContainers(ctx, false) | |
| 409 | + | if err != nil { | |
| 410 | + | log.Warn().Err(err).Msg("failed to list running containers for existing container resolution") | |
| 411 | + | return nil, false | |
| 412 | + | } | |
| 413 | + | ||
| 414 | + | for _, c := range running { | |
| 415 | + | if c.Name == expectedName && c.Labels[domain.LabelManaged] == "true" { | |
| 416 | + | log.Info(). | |
| 417 | + | Str("container_id", c.ID). | |
| 418 | + | Str("container_name", c.Name). | |
| 419 | + | Msg("resolved existing container from runtime (in-memory state was stale)") | |
| 420 | + | ||
| 421 | + | // Update in-memory state so subsequent lookups are fast | |
| 422 | + | s.mu.Lock() | |
| 423 | + | s.containers[domainName] = c | |
| 424 | + | s.managedCount++ | |
| 425 | + | s.mu.Unlock() | |
| 426 | + | ||
| 427 | + | return c, true | |
| 428 | + | } | |
| 429 | + | } | |
| 430 | + | ||
| 431 | + | return nil, false | |
| 432 | + | } | |
| 433 | + | ||
| 391 | 434 | func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) { | |
| 392 | 435 | log := zerowrap.FromCtx(ctx) | |
| 393 | 436 | ||
| @@ -276,7 +276,7 @@ func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Conta | |||||
| 276 | } | 276 | } | ||
| 277 | }() | 277 | }() | ||
| 278 | 278 | ||||
| 279 | - | existing, hasExisting := s.getTrackedContainer(route.Domain) | 279 | + | existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain) |
| 280 | 280 | ||||
| 281 | resources, err := s.prepareDeployResources(ctx, route, existing) | 281 | resources, err := s.prepareDeployResources(ctx, route, existing) | ||
| 282 | if err != nil { | 282 | if err != nil { | ||
| @@ -388,6 +388,49 @@ func (s *Service) getTrackedContainer(domainName string) (*domain.Container, boo | |||||
| 388 | return container, ok | 388 | return container, ok | ||
| 389 | } | 389 | } | ||
| 390 | 390 | ||||
| 391 | + | // resolveExistingContainer returns the currently running container for a domain. | |||
| 392 | + | // It first checks the in-memory map (fast path). If not found, it queries the | |||
| 393 | + | // runtime for a running container with matching name and managed label. This | |||
| 394 | + | // handles cases where Gordon restarted and in-memory state is stale. | |||
| 395 | + | func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) { | |||
| 396 | + | // Fast path: check in-memory state | |||
| 397 | + | s.mu.RLock() | |||
| 398 | + | container, ok := s.containers[domainName] | |||
| 399 | + | s.mu.RUnlock() | |||
| 400 | + | if ok { | |||
| 401 | + | return container, true | |||
| 402 | + | } | |||
| 403 | + | ||||
| 404 | + | // Slow path: query runtime for running containers | |||
| 405 | + | log := zerowrap.FromCtx(ctx) | |||
| 406 | + | expectedName := fmt.Sprintf("gordon-%s", domainName) | |||
| 407 | + | ||||
| 408 | + | running, err := s.runtime.ListContainers(ctx, false) | |||
| 409 | + | if err != nil { | |||
| 410 | + | log.Warn().Err(err).Msg("failed to list running containers for existing container resolution") | |||
| 411 | + | return nil, false | |||
| 412 | + | } | |||
| 413 | + | ||||
| 414 | + | for _, c := range running { | |||
| 415 | + | if c.Name == expectedName && c.Labels[domain.LabelManaged] == "true" { | |||
| 416 | + | log.Info(). | |||
| 417 | + | Str("container_id", c.ID). | |||
| 418 | + | Str("container_name", c.Name). | |||
| 419 | + | Msg("resolved existing container from runtime (in-memory state was stale)") | |||
| 420 | + | ||||
| 421 | + | // Update in-memory state so subsequent lookups are fast | |||
| 422 | + | s.mu.Lock() | |||
| 423 | + | s.containers[domainName] = c | |||
| 424 | + | s.managedCount++ | |||
| 425 | + | s.mu.Unlock() | |||
| 426 | + | ||||
| 427 | + | return c, true | |||
| 428 | + | } | |||
| 429 | + | } | |||
| 430 | + | ||||
| 431 | + | return nil, false | |||
| 432 | + | } | |||
| 433 | + | ||||
| 391 | func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) { | 434 | func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) { | ||
| 392 | log := zerowrap.FromCtx(ctx) | 435 | log := zerowrap.FromCtx(ctx) | ||
| 393 | 436 | ||||
M internal/usecase/container/service_test.go +145 -3
| @@ -198,6 +198,9 @@ func TestService_Deploy_Success(t *testing.T) { | |||
| 198 | 198 | Image: "myapp:latest", | |
| 199 | 199 | } | |
| 200 | 200 | ||
| 201 | + | // No container in memory — runtime resolution returns nothing | |
| 202 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |
| 203 | + | ||
| 201 | 204 | // Setup mocks - no orphaned containers | |
| 202 | 205 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |
| 203 | 206 | ||
| @@ -275,6 +278,9 @@ func TestService_Deploy_ReadinessRecoveryWindow_AllowsTransientFlap(t *testing.T | |||
| 275 | 278 | Image: "myapp:latest", | |
| 276 | 279 | } | |
| 277 | 280 | ||
| 281 | + | // No container in memory — runtime resolution returns nothing | |
| 282 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |
| 283 | + | ||
| 278 | 284 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |
| 279 | 285 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | |
| 280 | 286 | runtime.EXPECT().PullImage(mock.Anything, "myapp:latest").Return(nil) | |
| @@ -329,6 +335,9 @@ func TestService_Deploy_ImagePullFailure(t *testing.T) { | |||
| 329 | 335 | Image: "myapp:latest", | |
| 330 | 336 | } | |
| 331 | 337 | ||
| 338 | + | // No container in memory — runtime resolution returns nothing | |
| 339 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |
| 340 | + | ||
| 332 | 341 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |
| 333 | 342 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | |
| 334 | 343 | runtime.EXPECT().PullImage(mock.Anything, "myapp:latest").Return(errors.New("image not found")) | |
| @@ -471,10 +480,109 @@ func TestService_Deploy_ReplacesExistingContainer(t *testing.T) { | |||
| 471 | 480 | ||
| 472 | 481 | assert.NoError(t, err) | |
| 473 | 482 | assert.Equal(t, "new-container", result.ID) | |
| 483 | + | } | |
| 484 | + | ||
| 485 | + | // TestService_Deploy_ResolvesExistingFromRuntime_WhenMemoryStale verifies that | |
| 486 | + | // when in-memory state has no tracked container for a domain, Deploy queries | |
| 487 | + | // the runtime and discovers the running container. This prevents orphan cleanup | |
| 488 | + | // from killing the real active container after a Gordon restart. | |
| 489 | + | func TestService_Deploy_ResolvesExistingFromRuntime_WhenMemoryStale(t *testing.T) { | |
| 490 | + | runtime := mocks.NewMockContainerRuntime(t) | |
| 491 | + | envLoader := mocks.NewMockEnvLoader(t) | |
| 492 | + | eventBus := mocks.NewMockEventPublisher(t) | |
| 493 | + | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |
| 494 | + | ||
| 495 | + | config := Config{ | |
| 496 | + | ReadinessDelay: time.Millisecond, | |
| 497 | + | DrainDelay: time.Millisecond, | |
| 498 | + | } | |
| 499 | + | svc := NewService(runtime, envLoader, eventBus, nil, config) | |
| 500 | + | svc.SetProxyCacheInvalidator(cacheInvalidator) | |
| 501 | + | ctx := testContext() | |
| 502 | + | ||
| 503 | + | // NO tracked container in memory — simulates Gordon restart | |
| 504 | + | // (svc.containers is empty for "test.example.com") | |
| 505 | + | ||
| 506 | + | route := domain.Route{ | |
| 507 | + | Domain: "test.example.com", | |
| 508 | + | Image: "myapp:v2", | |
| 509 | + | } | |
| 510 | + | ||
| 511 | + | // resolveExistingContainer should query running containers from runtime | |
| 512 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{ | |
| 513 | + | { | |
| 514 | + | ID: "runtime-active-container", | |
| 515 | + | Name: "gordon-test.example.com", | |
| 516 | + | Status: "running", | |
| 517 | + | Labels: map[string]string{ | |
| 518 | + | domain.LabelDomain: "test.example.com", | |
| 519 | + | domain.LabelManaged: "true", | |
| 520 | + | }, | |
| 521 | + | }, | |
| 522 | + | }, nil) | |
| 523 | + | ||
| 524 | + | // Orphan cleanup lists ALL containers (including stopped) — the runtime-active | |
| 525 | + | // container should be skipped because it is now recognized as the existing one | |
| 526 | + | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{ | |
| 527 | + | { | |
| 528 | + | ID: "runtime-active-container", | |
| 529 | + | Name: "gordon-test.example.com", | |
| 530 | + | Status: "running", | |
| 531 | + | Labels: map[string]string{ | |
| 532 | + | domain.LabelDomain: "test.example.com", | |
| 533 | + | domain.LabelManaged: "true", | |
| 534 | + | }, | |
| 535 | + | }, | |
| 536 | + | }, nil) | |
| 537 | + | ||
| 538 | + | // Image operations | |
| 539 | + | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | |
| 540 | + | runtime.EXPECT().PullImage(mock.Anything, "myapp:v2").Return(nil) | |
| 541 | + | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:v2").Return([]int{8080}, nil) | |
| 542 | + | ||
| 543 | + | // Environment | |
| 544 | + | envLoader.EXPECT().LoadEnv(mock.Anything, "test.example.com").Return([]string{}, nil) | |
| 545 | + | runtime.EXPECT().InspectImageEnv(mock.Anything, "myapp:v2").Return([]string{}, nil) | |
| 474 | 546 | ||
| 475 | - | // Ensure the service's tracked container entry now points to the new container. | |
| 476 | - | tracked, ok := svc.Get(ctx, "test.example.com") | |
| 477 | - | assert.True(t, ok) | |
| 547 | + | // Create new container with -new suffix (zero-downtime: existing was resolved from runtime) | |
| 548 | + | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} | |
| 549 | + | runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool { | |
| 550 | + | return cfg.Name == "gordon-test.example.com-new" | |
| 551 | + | })).Return(newContainer, nil) | |
| 552 | + | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |
| 553 | + | ||
| 554 | + | // Wait for ready | |
| 555 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |
| 556 | + | ||
| 557 | + | // Inspect after ready | |
| 558 | + | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |
| 559 | + | ID: "new-container", | |
| 560 | + | Status: "running", | |
| 561 | + | }, nil) | |
| 562 | + | ||
| 563 | + | // Publish event | |
| 564 | + | eventBus.EXPECT().Publish(domain.EventContainerDeployed, mock.AnythingOfType("*domain.ContainerEventPayload")).Return(nil) | |
| 565 | + | ||
| 566 | + | // Synchronous cache invalidation | |
| 567 | + | cacheInvalidator.EXPECT().InvalidateTarget(mock.Anything, "test.example.com").Return() | |
| 568 | + | ||
| 569 | + | // AFTER new container is ready: stop and remove old runtime-discovered container | |
| 570 | + | // This is the key assertion — StopContainer on the runtime-active container | |
| 571 | + | // happens during finalizePreviousContainer, NOT during orphan cleanup | |
| 572 | + | runtime.EXPECT().StopContainer(mock.Anything, "runtime-active-container").Return(nil) | |
| 573 | + | runtime.EXPECT().RemoveContainer(mock.Anything, "runtime-active-container", true).Return(nil) | |
| 574 | + | ||
| 575 | + | // Rename new container to canonical name | |
| 576 | + | runtime.EXPECT().RenameContainer(mock.Anything, "new-container", "gordon-test.example.com").Return(nil) | |
| 577 | + | ||
| 578 | + | result, err := svc.Deploy(ctx, route) | |
| 579 | + | ||
| 580 | + | assert.NoError(t, err) | |
| 581 | + | assert.Equal(t, "new-container", result.ID) | |
| 582 | + | ||
| 583 | + | // Verify new container is now tracked | |
| 584 | + | tracked, exists := svc.Get(ctx, "test.example.com") | |
| 585 | + | assert.True(t, exists) | |
| 478 | 586 | assert.Equal(t, "new-container", tracked.ID) | |
| 479 | 587 | } | |
| 480 | 588 | ||
| @@ -555,6 +663,9 @@ func TestService_Deploy_WithNetworkIsolation(t *testing.T) { | |||
| 555 | 663 | Image: "myapp:latest", | |
| 556 | 664 | } | |
| 557 | 665 | ||
| 666 | + | // No container in memory — runtime resolution returns nothing | |
| 667 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |
| 668 | + | ||
| 558 | 669 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |
| 559 | 670 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp:latest"}, nil) | |
| 560 | 671 | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:latest").Return([]int{8080}, nil) | |
| @@ -607,6 +718,9 @@ func TestService_Deploy_WithVolumeAutoCreate(t *testing.T) { | |||
| 607 | 718 | Image: "myapp:latest", | |
| 608 | 719 | } | |
| 609 | 720 | ||
| 721 | + | // No container in memory — runtime resolution returns nothing | |
| 722 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |
| 723 | + | ||
| 610 | 724 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |
| 611 | 725 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp:latest"}, nil) | |
| 612 | 726 | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:latest").Return([]int{8080}, nil) | |
| @@ -1324,6 +1438,9 @@ func TestService_Deploy_InternalDeployForcesPull(t *testing.T) { | |||
| 1324 | 1438 | Image: "myapp:latest", | |
| 1325 | 1439 | } | |
| 1326 | 1440 | ||
| 1441 | + | // No container in memory — runtime resolution returns nothing | |
| 1442 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |
| 1443 | + | ||
| 1327 | 1444 | // Setup mocks - no orphaned containers | |
| 1328 | 1445 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |
| 1329 | 1446 | ||
| @@ -1390,6 +1507,9 @@ func TestService_AutoStart_StartsNewContainers(t *testing.T) { | |||
| 1390 | 1507 | {Domain: "app2.example.com", Image: "myapp2:latest"}, | |
| 1391 | 1508 | } | |
| 1392 | 1509 | ||
| 1510 | + | // No container in memory — runtime resolution returns nothing (one per route) | |
| 1511 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil).Times(2) | |
| 1512 | + | ||
| 1393 | 1513 | // Setup mocks for route deployments | |
| 1394 | 1514 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2) | |
| 1395 | 1515 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp1:latest", "myapp2:latest"}, nil).Times(2) | |
| @@ -1438,6 +1558,9 @@ func TestService_AutoStart_SkipsExistingContainers(t *testing.T) { | |||
| 1438 | 1558 | {Domain: "app2.example.com", Image: "myapp2:latest"}, // New route | |
| 1439 | 1559 | } | |
| 1440 | 1560 | ||
| 1561 | + | // No container in memory for app2 — runtime resolution returns nothing | |
| 1562 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil).Once() | |
| 1563 | + | ||
| 1441 | 1564 | // Only deploy for app2 (app1 is skipped). Readiness is skipped — no IsContainerRunning. | |
| 1442 | 1565 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Once() | |
| 1443 | 1566 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp2:latest"}, nil).Once() | |
| @@ -1472,6 +1595,9 @@ func TestService_AutoStart_HandlesDeployErrors(t *testing.T) { | |||
| 1472 | 1595 | {Domain: "app1.example.com", Image: "myapp1:latest"}, | |
| 1473 | 1596 | } | |
| 1474 | 1597 | ||
| 1598 | + | // No container in memory — runtime resolution returns nothing | |
| 1599 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |
| 1600 | + | ||
| 1475 | 1601 | // Setup mocks for failure | |
| 1476 | 1602 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | |
| 1477 | 1603 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | |
| @@ -1504,6 +1630,9 @@ func TestService_AutoStart_UsesInternalDeployContext(t *testing.T) { | |||
| 1504 | 1630 | {Domain: "app1.example.com", Image: "reg.example.com/myapp:latest"}, | |
| 1505 | 1631 | } | |
| 1506 | 1632 | ||
| 1633 | + | // No container in memory — runtime resolution returns nothing | |
| 1634 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |
| 1635 | + | ||
| 1507 | 1636 | // Key assertion: PullImage should be called with localhost:5000 rewrite, | |
| 1508 | 1637 | // NOT the original reg.example.com/myapp:latest. | |
| 1509 | 1638 | // Readiness is skipped — no IsContainerRunning calls. | |
| @@ -1640,6 +1769,16 @@ func TestService_Deploy_OrphanCleanupRemovesTrueOrphans(t *testing.T) { | |||
| 1640 | 1769 | Image: "myapp:v1", | |
| 1641 | 1770 | } | |
| 1642 | 1771 | ||
| 1772 | + | // No container in memory — runtime resolution finds no managed container | |
| 1773 | + | // (the orphan lacks LabelManaged so it won't match) | |
| 1774 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{ | |
| 1775 | + | { | |
| 1776 | + | ID: "orphan-container", | |
| 1777 | + | Name: "gordon-test.example.com", | |
| 1778 | + | Status: "running", | |
| 1779 | + | }, | |
| 1780 | + | }, nil) | |
| 1781 | + | ||
| 1643 | 1782 | // ListContainers returns an orphaned container (same name, but not tracked) | |
| 1644 | 1783 | // This could happen if Gordon crashed and restarted, or container was created manually | |
| 1645 | 1784 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{ | |
| @@ -1982,6 +2121,9 @@ func TestService_Deploy_ConcurrentSameDomain(t *testing.T) { | |||
| 1982 | 2121 | // First deploy: no existing container → creates gordon-test.example.com | |
| 1983 | 2122 | // Second deploy: sees first container → creates gordon-test.example.com-new | |
| 1984 | 2123 | ||
| 2124 | + | // First deploy resolves from runtime (no container in memory yet) | |
| 2125 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil).Once() | |
| 2126 | + | ||
| 1985 | 2127 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2) | |
| 1986 | 2128 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp:latest"}, nil).Times(2) | |
| 1987 | 2129 | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:latest").Return([]int{8080}, nil).Times(2) | |
| @@ -198,6 +198,9 @@ func TestService_Deploy_Success(t *testing.T) { | |||||
| 198 | Image: "myapp:latest", | 198 | Image: "myapp:latest", | ||
| 199 | } | 199 | } | ||
| 200 | 200 | ||||
| 201 | + | // No container in memory — runtime resolution returns nothing | |||
| 202 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |||
| 203 | + | ||||
| 201 | // Setup mocks - no orphaned containers | 204 | // Setup mocks - no orphaned containers | ||
| 202 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | 205 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | ||
| 203 | 206 | ||||
| @@ -275,6 +278,9 @@ func TestService_Deploy_ReadinessRecoveryWindow_AllowsTransientFlap(t *testing.T | |||||
| 275 | Image: "myapp:latest", | 278 | Image: "myapp:latest", | ||
| 276 | } | 279 | } | ||
| 277 | 280 | ||||
| 281 | + | // No container in memory — runtime resolution returns nothing | |||
| 282 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |||
| 283 | + | ||||
| 278 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | 284 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | ||
| 279 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | 285 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | ||
| 280 | runtime.EXPECT().PullImage(mock.Anything, "myapp:latest").Return(nil) | 286 | runtime.EXPECT().PullImage(mock.Anything, "myapp:latest").Return(nil) | ||
| @@ -329,6 +335,9 @@ func TestService_Deploy_ImagePullFailure(t *testing.T) { | |||||
| 329 | Image: "myapp:latest", | 335 | Image: "myapp:latest", | ||
| 330 | } | 336 | } | ||
| 331 | 337 | ||||
| 338 | + | // No container in memory — runtime resolution returns nothing | |||
| 339 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |||
| 340 | + | ||||
| 332 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | 341 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | ||
| 333 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | 342 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | ||
| 334 | runtime.EXPECT().PullImage(mock.Anything, "myapp:latest").Return(errors.New("image not found")) | 343 | runtime.EXPECT().PullImage(mock.Anything, "myapp:latest").Return(errors.New("image not found")) | ||
| @@ -471,10 +480,109 @@ func TestService_Deploy_ReplacesExistingContainer(t *testing.T) { | |||||
| 471 | 480 | ||||
| 472 | assert.NoError(t, err) | 481 | assert.NoError(t, err) | ||
| 473 | assert.Equal(t, "new-container", result.ID) | 482 | assert.Equal(t, "new-container", result.ID) | ||
| 483 | + | } | |||
| 484 | + | ||||
| 485 | + | // TestService_Deploy_ResolvesExistingFromRuntime_WhenMemoryStale verifies that | |||
| 486 | + | // when in-memory state has no tracked container for a domain, Deploy queries | |||
| 487 | + | // the runtime and discovers the running container. This prevents orphan cleanup | |||
| 488 | + | // from killing the real active container after a Gordon restart. | |||
| 489 | + | func TestService_Deploy_ResolvesExistingFromRuntime_WhenMemoryStale(t *testing.T) { | |||
| 490 | + | runtime := mocks.NewMockContainerRuntime(t) | |||
| 491 | + | envLoader := mocks.NewMockEnvLoader(t) | |||
| 492 | + | eventBus := mocks.NewMockEventPublisher(t) | |||
| 493 | + | cacheInvalidator := mocks.NewMockProxyCacheInvalidator(t) | |||
| 494 | + | ||||
| 495 | + | config := Config{ | |||
| 496 | + | ReadinessDelay: time.Millisecond, | |||
| 497 | + | DrainDelay: time.Millisecond, | |||
| 498 | + | } | |||
| 499 | + | svc := NewService(runtime, envLoader, eventBus, nil, config) | |||
| 500 | + | svc.SetProxyCacheInvalidator(cacheInvalidator) | |||
| 501 | + | ctx := testContext() | |||
| 502 | + | ||||
| 503 | + | // NO tracked container in memory — simulates Gordon restart | |||
| 504 | + | // (svc.containers is empty for "test.example.com") | |||
| 505 | + | ||||
| 506 | + | route := domain.Route{ | |||
| 507 | + | Domain: "test.example.com", | |||
| 508 | + | Image: "myapp:v2", | |||
| 509 | + | } | |||
| 510 | + | ||||
| 511 | + | // resolveExistingContainer should query running containers from runtime | |||
| 512 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{ | |||
| 513 | + | { | |||
| 514 | + | ID: "runtime-active-container", | |||
| 515 | + | Name: "gordon-test.example.com", | |||
| 516 | + | Status: "running", | |||
| 517 | + | Labels: map[string]string{ | |||
| 518 | + | domain.LabelDomain: "test.example.com", | |||
| 519 | + | domain.LabelManaged: "true", | |||
| 520 | + | }, | |||
| 521 | + | }, | |||
| 522 | + | }, nil) | |||
| 523 | + | ||||
| 524 | + | // Orphan cleanup lists ALL containers (including stopped) — the runtime-active | |||
| 525 | + | // container should be skipped because it is now recognized as the existing one | |||
| 526 | + | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{ | |||
| 527 | + | { | |||
| 528 | + | ID: "runtime-active-container", | |||
| 529 | + | Name: "gordon-test.example.com", | |||
| 530 | + | Status: "running", | |||
| 531 | + | Labels: map[string]string{ | |||
| 532 | + | domain.LabelDomain: "test.example.com", | |||
| 533 | + | domain.LabelManaged: "true", | |||
| 534 | + | }, | |||
| 535 | + | }, | |||
| 536 | + | }, nil) | |||
| 537 | + | ||||
| 538 | + | // Image operations | |||
| 539 | + | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | |||
| 540 | + | runtime.EXPECT().PullImage(mock.Anything, "myapp:v2").Return(nil) | |||
| 541 | + | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:v2").Return([]int{8080}, nil) | |||
| 542 | + | ||||
| 543 | + | // Environment | |||
| 544 | + | envLoader.EXPECT().LoadEnv(mock.Anything, "test.example.com").Return([]string{}, nil) | |||
| 545 | + | runtime.EXPECT().InspectImageEnv(mock.Anything, "myapp:v2").Return([]string{}, nil) | |||
| 474 | 546 | ||||
| 475 | - | // Ensure the service's tracked container entry now points to the new container. | 547 | + | // Create new container with -new suffix (zero-downtime: existing was resolved from runtime) |
| 476 | - | tracked, ok := svc.Get(ctx, "test.example.com") | 548 | + | newContainer := &domain.Container{ID: "new-container", Name: "gordon-test.example.com-new", Status: "created"} |
| 477 | - | assert.True(t, ok) | 549 | + | runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool { |
| 550 | + | return cfg.Name == "gordon-test.example.com-new" | |||
| 551 | + | })).Return(newContainer, nil) | |||
| 552 | + | runtime.EXPECT().StartContainer(mock.Anything, "new-container").Return(nil) | |||
| 553 | + | ||||
| 554 | + | // Wait for ready | |||
| 555 | + | runtime.EXPECT().IsContainerRunning(mock.Anything, "new-container").Return(true, nil).Times(2) | |||
| 556 | + | ||||
| 557 | + | // Inspect after ready | |||
| 558 | + | runtime.EXPECT().InspectContainer(mock.Anything, "new-container").Return(&domain.Container{ | |||
| 559 | + | ID: "new-container", | |||
| 560 | + | Status: "running", | |||
| 561 | + | }, nil) | |||
| 562 | + | ||||
| 563 | + | // Publish event | |||
| 564 | + | eventBus.EXPECT().Publish(domain.EventContainerDeployed, mock.AnythingOfType("*domain.ContainerEventPayload")).Return(nil) | |||
| 565 | + | ||||
| 566 | + | // Synchronous cache invalidation | |||
| 567 | + | cacheInvalidator.EXPECT().InvalidateTarget(mock.Anything, "test.example.com").Return() | |||
| 568 | + | ||||
| 569 | + | // AFTER new container is ready: stop and remove old runtime-discovered container | |||
| 570 | + | // This is the key assertion — StopContainer on the runtime-active container | |||
| 571 | + | // happens during finalizePreviousContainer, NOT during orphan cleanup | |||
| 572 | + | runtime.EXPECT().StopContainer(mock.Anything, "runtime-active-container").Return(nil) | |||
| 573 | + | runtime.EXPECT().RemoveContainer(mock.Anything, "runtime-active-container", true).Return(nil) | |||
| 574 | + | ||||
| 575 | + | // Rename new container to canonical name | |||
| 576 | + | runtime.EXPECT().RenameContainer(mock.Anything, "new-container", "gordon-test.example.com").Return(nil) | |||
| 577 | + | ||||
| 578 | + | result, err := svc.Deploy(ctx, route) | |||
| 579 | + | ||||
| 580 | + | assert.NoError(t, err) | |||
| 581 | + | assert.Equal(t, "new-container", result.ID) | |||
| 582 | + | ||||
| 583 | + | // Verify new container is now tracked | |||
| 584 | + | tracked, exists := svc.Get(ctx, "test.example.com") | |||
| 585 | + | assert.True(t, exists) | |||
| 478 | assert.Equal(t, "new-container", tracked.ID) | 586 | assert.Equal(t, "new-container", tracked.ID) | ||
| 479 | } | 587 | } | ||
| 480 | 588 | ||||
| @@ -555,6 +663,9 @@ func TestService_Deploy_WithNetworkIsolation(t *testing.T) { | |||||
| 555 | Image: "myapp:latest", | 663 | Image: "myapp:latest", | ||
| 556 | } | 664 | } | ||
| 557 | 665 | ||||
| 666 | + | // No container in memory — runtime resolution returns nothing | |||
| 667 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |||
| 668 | + | ||||
| 558 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | 669 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | ||
| 559 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp:latest"}, nil) | 670 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp:latest"}, nil) | ||
| 560 | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:latest").Return([]int{8080}, nil) | 671 | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:latest").Return([]int{8080}, nil) | ||
| @@ -607,6 +718,9 @@ func TestService_Deploy_WithVolumeAutoCreate(t *testing.T) { | |||||
| 607 | Image: "myapp:latest", | 718 | Image: "myapp:latest", | ||
| 608 | } | 719 | } | ||
| 609 | 720 | ||||
| 721 | + | // No container in memory — runtime resolution returns nothing | |||
| 722 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |||
| 723 | + | ||||
| 610 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | 724 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | ||
| 611 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp:latest"}, nil) | 725 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp:latest"}, nil) | ||
| 612 | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:latest").Return([]int{8080}, nil) | 726 | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:latest").Return([]int{8080}, nil) | ||
| @@ -1324,6 +1438,9 @@ func TestService_Deploy_InternalDeployForcesPull(t *testing.T) { | |||||
| 1324 | Image: "myapp:latest", | 1438 | Image: "myapp:latest", | ||
| 1325 | } | 1439 | } | ||
| 1326 | 1440 | ||||
| 1441 | + | // No container in memory — runtime resolution returns nothing | |||
| 1442 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |||
| 1443 | + | ||||
| 1327 | // Setup mocks - no orphaned containers | 1444 | // Setup mocks - no orphaned containers | ||
| 1328 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | 1445 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | ||
| 1329 | 1446 | ||||
| @@ -1390,6 +1507,9 @@ func TestService_AutoStart_StartsNewContainers(t *testing.T) { | |||||
| 1390 | {Domain: "app2.example.com", Image: "myapp2:latest"}, | 1507 | {Domain: "app2.example.com", Image: "myapp2:latest"}, | ||
| 1391 | } | 1508 | } | ||
| 1392 | 1509 | ||||
| 1510 | + | // No container in memory — runtime resolution returns nothing (one per route) | |||
| 1511 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil).Times(2) | |||
| 1512 | + | ||||
| 1393 | // Setup mocks for route deployments | 1513 | // Setup mocks for route deployments | ||
| 1394 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2) | 1514 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2) | ||
| 1395 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp1:latest", "myapp2:latest"}, nil).Times(2) | 1515 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp1:latest", "myapp2:latest"}, nil).Times(2) | ||
| @@ -1438,6 +1558,9 @@ func TestService_AutoStart_SkipsExistingContainers(t *testing.T) { | |||||
| 1438 | {Domain: "app2.example.com", Image: "myapp2:latest"}, // New route | 1558 | {Domain: "app2.example.com", Image: "myapp2:latest"}, // New route | ||
| 1439 | } | 1559 | } | ||
| 1440 | 1560 | ||||
| 1561 | + | // No container in memory for app2 — runtime resolution returns nothing | |||
| 1562 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil).Once() | |||
| 1563 | + | ||||
| 1441 | // Only deploy for app2 (app1 is skipped). Readiness is skipped — no IsContainerRunning. | 1564 | // Only deploy for app2 (app1 is skipped). Readiness is skipped — no IsContainerRunning. | ||
| 1442 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Once() | 1565 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Once() | ||
| 1443 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp2:latest"}, nil).Once() | 1566 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp2:latest"}, nil).Once() | ||
| @@ -1472,6 +1595,9 @@ func TestService_AutoStart_HandlesDeployErrors(t *testing.T) { | |||||
| 1472 | {Domain: "app1.example.com", Image: "myapp1:latest"}, | 1595 | {Domain: "app1.example.com", Image: "myapp1:latest"}, | ||
| 1473 | } | 1596 | } | ||
| 1474 | 1597 | ||||
| 1598 | + | // No container in memory — runtime resolution returns nothing | |||
| 1599 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |||
| 1600 | + | ||||
| 1475 | // Setup mocks for failure | 1601 | // Setup mocks for failure | ||
| 1476 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | 1602 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil) | ||
| 1477 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | 1603 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{}, nil) | ||
| @@ -1504,6 +1630,9 @@ func TestService_AutoStart_UsesInternalDeployContext(t *testing.T) { | |||||
| 1504 | {Domain: "app1.example.com", Image: "reg.example.com/myapp:latest"}, | 1630 | {Domain: "app1.example.com", Image: "reg.example.com/myapp:latest"}, | ||
| 1505 | } | 1631 | } | ||
| 1506 | 1632 | ||||
| 1633 | + | // No container in memory — runtime resolution returns nothing | |||
| 1634 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil) | |||
| 1635 | + | ||||
| 1507 | // Key assertion: PullImage should be called with localhost:5000 rewrite, | 1636 | // Key assertion: PullImage should be called with localhost:5000 rewrite, | ||
| 1508 | // NOT the original reg.example.com/myapp:latest. | 1637 | // NOT the original reg.example.com/myapp:latest. | ||
| 1509 | // Readiness is skipped — no IsContainerRunning calls. | 1638 | // Readiness is skipped — no IsContainerRunning calls. | ||
| @@ -1640,6 +1769,16 @@ func TestService_Deploy_OrphanCleanupRemovesTrueOrphans(t *testing.T) { | |||||
| 1640 | Image: "myapp:v1", | 1769 | Image: "myapp:v1", | ||
| 1641 | } | 1770 | } | ||
| 1642 | 1771 | ||||
| 1772 | + | // No container in memory — runtime resolution finds no managed container | |||
| 1773 | + | // (the orphan lacks LabelManaged so it won't match) | |||
| 1774 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{ | |||
| 1775 | + | { | |||
| 1776 | + | ID: "orphan-container", | |||
| 1777 | + | Name: "gordon-test.example.com", | |||
| 1778 | + | Status: "running", | |||
| 1779 | + | }, | |||
| 1780 | + | }, nil) | |||
| 1781 | + | ||||
| 1643 | // ListContainers returns an orphaned container (same name, but not tracked) | 1782 | // ListContainers returns an orphaned container (same name, but not tracked) | ||
| 1644 | // This could happen if Gordon crashed and restarted, or container was created manually | 1783 | // This could happen if Gordon crashed and restarted, or container was created manually | ||
| 1645 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{ | 1784 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{ | ||
| @@ -1982,6 +2121,9 @@ func TestService_Deploy_ConcurrentSameDomain(t *testing.T) { | |||||
| 1982 | // First deploy: no existing container → creates gordon-test.example.com | 2121 | // First deploy: no existing container → creates gordon-test.example.com | ||
| 1983 | // Second deploy: sees first container → creates gordon-test.example.com-new | 2122 | // Second deploy: sees first container → creates gordon-test.example.com-new | ||
| 1984 | 2123 | ||||
| 2124 | + | // First deploy resolves from runtime (no container in memory yet) | |||
| 2125 | + | runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{}, nil).Once() | |||
| 2126 | + | ||||
| 1985 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2) | 2127 | runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2) | ||
| 1986 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp:latest"}, nil).Times(2) | 2128 | runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp:latest"}, nil).Times(2) | ||
| 1987 | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:latest").Return([]int{8080}, nil).Times(2) | 2129 | runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp:latest").Return([]int{8080}, nil).Times(2) | ||