1package deployment_test
2
3import (
4 "context"
5 "errors"
6 "io"
7 "strings"
8 "testing"
9 "time"
10
11 "github.com/bnema/zerowrap"
12 "github.com/stretchr/testify/assert"
13 "github.com/stretchr/testify/mock"
14 "github.com/stretchr/testify/require"
15
16 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
17 "github.com/bnema/gordon/internal/domain"
18 "github.com/bnema/gordon/internal/usecase/deployment"
19)
20
21func mockDeps(t *testing.T) (*outmocks.MockAppState, *outmocks.MockContainerRuntime, *outmocks.MockImageResolver, *outmocks.MockSecretProvider) {
22 t.Helper()
23 state := outmocks.NewMockAppState(t)
24 // Every app mutation reconciles an interrupted predecessor first. These
25 // tests exercise the mutation itself, so they see no interrupted
26 // operation; tests that seed one use their own expectation or a real
27 // store.
28 state.EXPECT().LoadLatestOperation(mock.Anything, mock.Anything).
29 Return(domain.AppOperation{}, false, nil).Maybe()
30 // ExecuteDeploy always reloads the claimed operation from the store by
31 // app/op identity. Mock-backed execution sees the freshly claimed
32 // non-terminal journal shape (a single pending preflight step) unless a
33 // test overrides it with expectStoredOperation.
34 state.EXPECT().LoadOperation(mock.Anything, mock.Anything, mock.Anything).
35 RunAndReturn(func(_ context.Context, app, opID string) (domain.AppOperation, error) {
36 return domain.AppOperation{
37 Kind: "deploy", App: app, Op: opID,
38 Steps: []domain.AppOperationStep{{ID: "preflight", State: domain.AppStepPending}},
39 }, nil
40 }).Maybe()
41 return state,
42 outmocks.NewMockContainerRuntime(t),
43 outmocks.NewMockImageResolver(t),
44 outmocks.NewMockSecretProvider(t)
45}
46
47// expectStoredOperation makes one store record authoritative for the next
48// ExecuteDeploy reload. The operation identity is echoed from the request so
49// an unkeyed claim matches too. It replaces the catch-all default installed
50// by mockDeps, letting a test prove that a terminal or mismatched store
51// record overrides a stale in-process journal.
52func expectStoredOperation(state *outmocks.MockAppState, op domain.AppOperation) {
53 kept := state.ExpectedCalls[:0]
54 for _, call := range state.ExpectedCalls {
55 if call.Method != "LoadOperation" {
56 kept = append(kept, call)
57 }
58 }
59 state.ExpectedCalls = kept
60 state.EXPECT().LoadOperation(mock.Anything, op.App, mock.Anything).
61 RunAndReturn(func(_ context.Context, app, opID string) (domain.AppOperation, error) {
62 op.App = app
63 op.Op = opID
64 return op, nil
65 })
66}
67
68func mockRevision() domain.AppDesiredRevision {
69 svc := webService()
70 svc.Image = "docker.io/example/web:1.4.2"
71 svc.Readiness.Timeout = time.Second
72 return domain.AppDesiredRevision{
73 Revision: "rev-1", App: "blog",
74 Spec: domain.AppSpec{Name: "blog", Env: map[string]string{}, Services: []domain.AppService{svc}},
75 }
76}
77
78// expectDeployPreflight wires recover + revision + digest + secrets +
79// image volumes + checkpoint + ownership + the two journal writes that
80// frame execution (preflight table, then per-service/outcome updates).
81func expectDeployPreflight(
82 state *outmocks.MockAppState,
83 runtime *outmocks.MockContainerRuntime,
84 images *outmocks.MockImageResolver,
85 secrets *outmocks.MockSecretProvider,
86 rev domain.AppDesiredRevision,
87) {
88 state.EXPECT().Recover(mock.Anything).Return(nil)
89 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
90 images.EXPECT().ResolveDigest(mock.Anything, rev.Spec.Services[0].Image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil)
91 secrets.EXPECT().GetSecret(mock.Anything, "gordon/apps/app-blog/web/database-url").Return("x", nil)
92 runtime.EXPECT().InspectImageVolumes(mock.Anything, rev.Spec.Services[0].Image).Return(nil, nil)
93 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil)
94 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
95 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
96}
97
98// expectNetworkProvision registers the two runtime calls every
99// create/recovery path makes when the incarnation network does not yet
100// exist: inspect existing networks (none), then create the private one.
101func expectNetworkProvision(runtime *outmocks.MockContainerRuntime, appID string, times int) {
102 runtime.EXPECT().ListNetworks(mock.Anything).Return([]*domain.NetworkInfo{}, nil).Times(times)
103 runtime.EXPECT().CreateNetwork(mock.Anything, domain.AppPrivateNetworkName("gordon", appID), mock.Anything).Return(nil).Times(times)
104}
105
106func TestDeploy_PreflightFailureReturnsJournaledOperation(t *testing.T) {
107 ctx := context.Background()
108 state, runtime, images, secrets := mockDeps(t)
109 rev := mockRevision()
110
111 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
112 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil).Once()
113 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil).Twice()
114 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil).Once()
115 images.EXPECT().ResolveDigest(mock.Anything, rev.Spec.Services[0].Image).Return("", assert.AnError).Once()
116 // The store record carries the revision the claim was started for.
117 expectStoredOperation(state, domain.AppOperation{
118 Kind: "deploy", App: "blog", InputRevision: rev.Revision,
119 Steps: []domain.AppOperationStep{{ID: "preflight", State: domain.AppStepPending}},
120 })
121
122 svc := deployment.NewService(deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets}, zerowrap.Default())
123 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
124
125 require.ErrorIs(t, err, domain.ErrAppImageUnresolvable)
126 require.NotNil(t, result)
127 assert.NotEmpty(t, result.Op)
128 assert.Equal(t, "blog", result.App)
129 assert.Equal(t, rev.Revision, result.Revision)
130 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
131}
132
133func TestDeploy_Mockery_PullsPinnedImageWithRegistryAuth(t *testing.T) {
134 ctx := context.Background()
135 state, runtime, images, secrets := mockDeps(t)
136 svcSpec := webService()
137 svcSpec.Image = "registry.example.com/blog/web:1.4.2"
138 svcSpec.HTTP = nil
139 svcSpec.Readiness = domain.AppReadiness{}
140 rev := domain.AppDesiredRevision{
141 Revision: "rev-1", App: "blog",
142 Spec: domain.AppSpec{Name: "blog", Env: map[string]string{}, Services: []domain.AppService{svcSpec}},
143 }
144
145 state.EXPECT().Recover(mock.Anything).Return(nil)
146 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
147 images.EXPECT().ResolveDigest(mock.Anything, svcSpec.Image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil)
148 secrets.EXPECT().GetSecret(mock.Anything, "gordon/apps/app-blog/web/database-url").Return("x", nil)
149 pullRef := "127.0.0.1:5000/blog/web@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
150 runtime.EXPECT().PullImageWithOptions(mock.Anything, domain.ImagePullRequest{Reference: pullRef, Username: "gordon", Password: "s3cret", Transport: domain.ImagePullTransportHTTP}).Return(nil).Once()
151 runtime.EXPECT().VerifyImageDigest(mock.Anything, pullRef, "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil).Once()
152 runtime.EXPECT().InspectImageVolumes(mock.Anything, "127.0.0.1:5000/blog/web@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Return(nil, nil)
153 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil)
154 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
155 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
156 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
157 App: "blog",
158 Services: map[string]domain.AppEffectiveService{},
159 }, true, nil)
160 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
161 expectNetworkProvision(runtime, "app-blog", 1)
162
163 // The pinned ref was pulled during preflight before image-volume inspection.
164 runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool {
165 return cfg.Image == "127.0.0.1:5000/blog/web@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" &&
166 assert.Equal(t, svcSpec.Command, cfg.Entrypoint) && assert.Empty(t, cfg.Cmd)
167 })).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
168 runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
169 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
170 // No TCP-capable interfaces (HTTP nil, readiness empty): no backend
171 // binds published, no GetContainerBackendBinds call.
172 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
173 App: "blog",
174 Services: map[string]domain.AppEffectiveService{},
175 }, true, nil)
176 state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil)
177 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
178
179 svc := deployment.NewService(
180 deployment.Deps{
181 State: state, Runtime: runtime, Images: images, Secrets: secrets,
182 Registry: deployment.RegistryConfig{
183 Domain: "registry.example.com",
184 PullAddress: "127.0.0.1:5000",
185 Username: "gordon",
186 Password: "s3cret",
187 },
188 },
189 zerowrap.Default(),
190 ).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
191 func(context.Context, string, string) (int, error) { return 200, nil },
192 func(context.Context, string) error { return nil },
193 ))
194
195 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
196 require.NoError(t, err)
197 require.NotNil(t, result)
198 assert.Equal(t, "deployed", result.Services["web"].Result)
199 assert.Empty(t, result.Services["web"].BackendBinds)
200}
201
202// TestDeploy_Mockery_HTTPSuccessRecordsLoopbackBinds proves the rootless-first
203// readiness path: the engine publishes the HTTP container port on
204// 127.0.0.1 ephemeral, probes the recorded loopback bind (never a
205// container IP), and records the binds for the active record + proxy.
206func TestDeploy_Mockery_HTTPSuccessRecordsLoopbackBinds(t *testing.T) {
207 ctx := context.Background()
208 state, runtime, images, secrets := mockDeps(t)
209 rev := mockRevision()
210
211 expectDeployPreflight(state, runtime, images, secrets, rev)
212 expectNetworkProvision(runtime, "app-blog", 1)
213 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
214 App: "blog", Services: map[string]domain.AppEffectiveService{},
215 }, true, nil)
216 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
217
218 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
219 runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
220 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
221 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
222 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
223 return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
224 })).Return(nil).Once()
225 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
226 App: "blog", Services: map[string]domain.AppEffectiveService{},
227 }, true, nil)
228 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(a domain.AppActive) bool {
229 return a.Services["web"].BackendBinds[8080] == 18080
230 })).Return(nil).Once()
231 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
232
233 var probedURL string
234 svc := deployment.NewService(
235 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
236 zerowrap.Default(),
237 ).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
238 func(_ context.Context, url, _ string) (int, error) { probedURL = url; return 200, nil },
239 func(context.Context, string) error { return nil },
240 ))
241
242 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
243 require.NoError(t, err)
244 require.NotNil(t, result)
245 assert.Equal(t, "deployed", result.Services["web"].Result)
246 assert.Equal(t, map[int]int{8080: 18080}, result.Services["web"].BackendBinds)
247 assert.Contains(t, probedURL, "127.0.0.1:18080", "readiness dials the loopback bind, never a container IP")
248 runtime.AssertNotCalled(t, "GetContainerNetworkInfo", mock.Anything, mock.Anything)
249}
250
251// TestDeploy_Mockery_HTTPOverlapPublishesBeforeRetiringOldGeneration proves
252// the overlap ordering of a stateless public HTTP service: the replacement is
253// created while the superseded container keeps running, ACTIVE names the
254// replacement next, and the superseded container is stopped and removed only
255// afterwards, releasing its backend claims.
256func TestDeploy_Mockery_HTTPOverlapPublishesBeforeRetiringOldGeneration(t *testing.T) {
257 ctx := context.Background()
258 state, runtime, images, secrets := mockDeps(t)
259 rev := mockRevision()
260
261 expectDeployPreflight(state, runtime, images, secrets, rev)
262 expectNetworkProvision(runtime, "app-blog", 1)
263 oldActive := domain.AppActive{
264 App: "blog",
265 Services: map[string]domain.AppEffectiveService{
266 "web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", BackendBinds: map[int]int{8080: 17000}},
267 },
268 }
269 state.EXPECT().LoadActive(mock.Anything, "blog").Return(oldActive, true, nil)
270 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
271
272 var createdAt, publishedAt, stoppedAt int
273 var step int
274 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
275 step++
276 createdAt = step
277 return &domain.Container{ID: "c-new", Name: "web"}, nil
278 }).Once()
279 runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
280 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
281 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
282 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
283 return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
284 })).Return(nil).Once()
285 var published domain.AppActive
286 state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, active domain.AppActive) error {
287 step++
288 publishedAt = step
289 published = active
290 return nil
291 }).Once()
292 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
293 expectRetirementMarker(state, "c-old")
294 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
295 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(func(context.Context, string, time.Duration) error {
296 step++
297 stoppedAt = step
298 return nil
299 }).Once()
300 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
301 // The superseded container's loopback claims are released once it is gone.
302 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
303
304 svc := deployment.NewService(
305 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
306 zerowrap.Default(),
307 ).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
308 func(context.Context, string, string) (int, error) { return 200, nil },
309 func(context.Context, string) error { return nil },
310 ))
311
312 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
313 require.NoError(t, err)
314 require.NotNil(t, result)
315 assert.Equal(t, "deployed", result.Services["web"].Result)
316 assert.Less(t, createdAt, publishedAt, "ACTIVE is published after the replacement exists")
317 assert.Less(t, publishedAt, stoppedAt, "the superseded generation is stopped only after the switch")
318 assert.Equal(t, "c-new", published.Services["web"].Container)
319 assert.Equal(t, map[int]int{8080: 18080}, published.Services["web"].BackendBinds,
320 "ACTIVE no longer references the superseded container's binds")
321 assert.Empty(t, result.CleanupWarnings)
322}
323
324// TestDeploy_Mockery_HTTPSequentialWhenServiceHasVolume proves a service with a
325// volume keeps the sequential ordering: the superseded container is stopped
326// and removed before the replacement is created, and ACTIVE names the
327// replacement only after it exists.
328func TestDeploy_Mockery_HTTPSequentialWhenServiceHasVolume(t *testing.T) {
329 ctx := context.Background()
330 state, runtime, images, secrets := mockDeps(t)
331 rev := statefulMockRevision()
332
333 expectDeployPreflight(state, runtime, images, secrets, rev)
334 runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil)
335 runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil)
336 expectNetworkProvision(runtime, "app-blog", 1)
337 active := domain.AppActive{
338 App: "blog",
339 Services: map[string]domain.AppEffectiveService{
340 "web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old"},
341 },
342 }
343 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
344 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
345 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
346
347 var replacedAt, createdAt, publishedAt int
348 var step int
349 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(func(context.Context, string, time.Duration) error {
350 step++
351 replacedAt = step
352 return nil
353 }).Once()
354 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
355 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
356 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
357 step++
358 createdAt = step
359 return &domain.Container{ID: "c-new", Name: "web"}, nil
360 }).Once()
361 runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
362 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
363 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
364 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
365 state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(func(context.Context, domain.AppActive) error {
366 step++
367 publishedAt = step
368 return nil
369 }).Once()
370 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
371 // The superseded generation stops being recovery-inhibited only
372 // once the replacement is published.
373 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
374
375 svc := deployment.NewService(
376 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
377 zerowrap.Default(),
378 ).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
379 func(context.Context, string, string) (int, error) { return 200, nil },
380 func(context.Context, string) error { return nil },
381 ))
382
383 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
384 require.NoError(t, err)
385 require.NotNil(t, result)
386 assert.Equal(t, "deployed", result.Services["web"].Result)
387 assert.Less(t, replacedAt, createdAt, "the superseded generation is gone before the replacement is created")
388 assert.Greater(t, publishedAt, createdAt, "ACTIVE is published after the replacement exists")
389 assert.Empty(t, result.CleanupWarnings)
390}
391
392// TestDeploy_Mockery_HTTPReadinessFailureKeepsOldGenerationServing proves a
393// failed overlap replacement leaves the generation that serves alone: the
394// superseded container is neither stopped nor removed nor released, the
395// unready candidate is removed, and nothing is published in its place.
396func TestDeploy_Mockery_HTTPReadinessFailureKeepsOldGenerationServing(t *testing.T) {
397 ctx := context.Background()
398 state, runtime, images, secrets := mockDeps(t)
399 rev := mockRevision()
400
401 expectDeployPreflight(state, runtime, images, secrets, rev)
402 expectNetworkProvision(runtime, "app-blog", 1)
403 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
404 App: "blog",
405 Services: map[string]domain.AppEffectiveService{
406 "web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old"},
407 },
408 }, true, nil)
409 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
410
411 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
412 runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
413 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
414 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
415 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
416 return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
417 })).Return(nil).Once()
418 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(nil).Once()
419 // The failed candidate's claims are released: a candidate that never
420 // became effective must not keep a loopback reservation.
421 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-new").Return(nil).Once()
422 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(io.NopCloser(strings.NewReader("boom\n")), nil).Once()
423
424 svc := deployment.NewService(
425 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
426 zerowrap.Default(),
427 ).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
428 func(context.Context, string, string) (int, error) { return 500, nil },
429 func(context.Context, string) error { return errors.New("refused") },
430 ))
431
432 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
433 require.Error(t, err)
434 require.NotNil(t, result)
435 assert.Equal(t, "failed", result.Services["web"].Result)
436 assert.NotContains(t, result.Services["web"].Error, "logs:", "raw logs must never be embedded in the public error")
437 require.Len(t, result.Services["web"].Diagnostics, 1, "failure diagnostics are kept separately")
438 assert.Equal(t, "boom", result.Services["web"].Diagnostics[0])
439 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
440 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", mock.Anything)
441 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
442 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
443 state.AssertNotCalled(t, "SaveRecoveryInhibition", mock.Anything, mock.Anything)
444 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-old")
445}
446
447func TestDeploy_Mockery_VolumeReplacementMarksUnsafe(t *testing.T) {
448 ctx := context.Background()
449 state, runtime, images, secrets := mockDeps(t)
450
451 // The GC barrier must be held across resource acquisition through
452 // the durable ownership publication that protects the volume.
453 barrier := outmocks.NewMockGCBarrier(t)
454 gcLease := outmocks.NewMockGCLease(t)
455 gcLeaseHeld := false
456 ownershipSaves := 0
457 barrier.EXPECT().AcquireShared(mock.Anything).Run(func(context.Context) {
458 gcLeaseHeld = true
459 }).Return(gcLease, nil).Once()
460 gcLease.EXPECT().Release().Run(func() {
461 gcLeaseHeld = false
462 }).Once()
463
464 svcSpec := webService()
465 svcSpec.HTTP = nil
466 svcSpec.TCP = []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 9000, Publish: "9000"}}
467 svcSpec.Readiness = domain.AppReadiness{Type: "none", Timeout: time.Second}
468 svcSpec.Volumes = []domain.AppVolume{{Name: "d", Path: "/data"}}
469 svcSpec.Secrets = map[string]string{}
470 rev := domain.AppDesiredRevision{
471 Revision: "rev-1", App: "blog",
472 Spec: domain.AppSpec{Name: "blog", Env: map[string]string{}, Services: []domain.AppService{svcSpec}},
473 }
474
475 state.EXPECT().Recover(mock.Anything).Return(nil)
476 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
477 images.EXPECT().ResolveDigest(mock.Anything, svcSpec.Image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil)
478 runtime.EXPECT().InspectImageVolumes(mock.Anything, svcSpec.Image).Return(nil, nil)
479 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil)
480 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-uuid-blog", Volumes: []domain.AppOwnedVolume{{Name: "d", Service: "web", RuntimeName: "gordon-blog--web--vol--d", State: domain.AppResourceAttached}}}, nil).Times(4)
481 runtime.EXPECT().VolumeExists(mock.Anything, "gordon-blog--web--vol--d").Return(true, nil)
482 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
483 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
484 App: "blog",
485 Services: map[string]domain.AppEffectiveService{
486 "web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old"},
487 },
488 }, true, nil)
489 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
490
491 var order []string
492 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(func(context.Context, string, time.Duration) error {
493 order = append(order, "stop-old")
494 return nil
495 }).Once()
496 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).RunAndReturn(func(context.Context, string, bool) error {
497 order = append(order, "remove-old")
498 return nil
499 }).Once()
500 // The superseded writer's confirmed disappearance releases its claims.
501 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
502 expectNetworkProvision(runtime, "app-uuid-blog", 1)
503 // The volume-owning replacement must be inhibited before it can
504 // write, and released only after the new generation is published.
505 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.MatchedBy(func(inhibition domain.AppRecoveryInhibition) bool {
506 return inhibition.App == "blog" && inhibition.Service == "web" &&
507 inhibition.ContainerID == "c-old" && inhibition.Reason == domain.AppInhibitReplacementPending
508 })).RunAndReturn(func(context.Context, domain.AppRecoveryInhibition) error {
509 order = append(order, "inhibit")
510 return nil
511 }).Once()
512 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
513 runtime.EXPECT().CreateVolume(mock.Anything, "gordon-blog--web--vol--d", mock.MatchedBy(func(labels map[string]string) bool {
514 return labels[domain.LabelApp] == "blog" &&
515 labels[domain.LabelAppID] == "app-uuid-blog" &&
516 labels[domain.LabelAppService] == "web" &&
517 labels[domain.LabelAppRevision] == "rev-1"
518 })).Run(func(context.Context, string, map[string]string) {
519 require.True(t, gcLeaseHeld, "volume creation must run inside the shared GC lease")
520 // The durable ownership reservation must already be written:
521 // a crash after this point leaves a protected record, never an
522 // unowned volume that prune could later adopt.
523 require.GreaterOrEqual(t, ownershipSaves, 1,
524 "ownership must be reserved before the runtime volume exists")
525 }).Return(nil).Once()
526 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
527 order = append(order, "create-new")
528 return &domain.Container{ID: "c-new", Name: "web"}, nil
529 }).Once()
530 runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
531 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
532 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 9000, HostPort: 19000, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
533 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
534 return len(claims) == 1 && claims[0].Port == 19000 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
535 })).Return(nil).Once()
536 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{}}, true, nil)
537 state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
538 restartUnsafeSaved := false
539 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil).Run(func(_ context.Context, ownership domain.AppOwnership) {
540 require.True(t, gcLeaseHeld, "ownership publication must run inside the shared GC lease")
541 ownershipSaves++
542 if ownership.Services["web"].RestartUnsafe {
543 restartUnsafeSaved = true
544 }
545 })
546
547 svc := deployment.NewService(
548 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
549 zerowrap.Default(),
550 ).WithProbeDeps(deployment.NewProbeDeps(runtime)).WithGCBarrier(barrier)
551
552 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
553 require.NoError(t, err)
554 require.NotNil(t, result)
555 assert.Equal(t, []string{"inhibit", "stop-old", "remove-old", "create-new"}, order,
556 "the durable inhibition is written before the superseded writer can be stopped, and the writer is gone before the replacement can write")
557 assert.True(t, result.Services["web"].RestartUnsafe)
558 assert.True(t, restartUnsafeSaved, "the terminal ownership write must record the restart-unsafe service")
559 assert.GreaterOrEqual(t, ownershipSaves, 2, "the volume reservation and the terminal ownership write must both be durable")
560 assert.False(t, gcLeaseHeld, "the shared GC lease must be released after publication")
561 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
562}
563
564// TestDeploy_Mockery_MixedTCPUDPPublishesBothLoopbacks proves a mixed
565// TCP+UDP service publishes both protocols on 127.0.0.1 ephemeral,
566// resolves them in one grouped inspection (same container port number
567// on both protocols yields two distinct binds), and persists both maps
568// with exact-protocol claims.
569func TestDeploy_Mockery_MixedTCPUDPPublishesBothLoopbacks(t *testing.T) {
570 ctx := context.Background()
571 state, runtime, images, secrets := mockDeps(t)
572 svcSpec := webService()
573 svcSpec.HTTP = nil
574 svcSpec.TCP = []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 9000, Publish: "9000"}}
575 svcSpec.UDP = []domain.AppUDPInterface{{Entrypoint: "udp", Port: 9000, Publish: "9000"}}
576 svcSpec.Readiness = domain.AppReadiness{Type: "none", Timeout: time.Second}
577 svcSpec.Secrets = map[string]string{}
578 rev := domain.AppDesiredRevision{
579 Revision: "rev-1", App: "blog",
580 Spec: domain.AppSpec{Name: "blog", Env: map[string]string{}, Services: []domain.AppService{svcSpec}},
581 }
582
583 state.EXPECT().Recover(mock.Anything).Return(nil)
584 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
585 images.EXPECT().ResolveDigest(mock.Anything, svcSpec.Image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil)
586 runtime.EXPECT().InspectImageVolumes(mock.Anything, svcSpec.Image).Return(nil, nil)
587 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil)
588 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
589 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
590 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
591 App: "blog", Services: map[string]domain.AppEffectiveService{},
592 }, true, nil)
593 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
594 expectNetworkProvision(runtime, "app-blog", 1)
595
596 runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool {
597 if len(cfg.PortPublishes) != 2 {
598 return false
599 }
600 byProto := map[domain.NetworkProtocol]domain.ContainerPortPublish{}
601 for _, publish := range cfg.PortPublishes {
602 if publish.HostIP != "127.0.0.1" || publish.HostPort != 0 || publish.ContainerPort != 9000 {
603 return false
604 }
605 byProto[publish.Protocol] = publish
606 }
607 return len(byProto) == 2
608 })).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
609 runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
610 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
611 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.MatchedBy(func(ports []domain.ContainerBackendPort) bool {
612 if len(ports) != 2 {
613 return false
614 }
615 seen := map[domain.ContainerBackendPort]bool{}
616 for _, port := range ports {
617 seen[port] = true
618 }
619 return seen[domain.ContainerBackendPort{ContainerPort: 9000, Protocol: domain.NetworkProtocolTCP}] &&
620 seen[domain.ContainerBackendPort{ContainerPort: 9000, Protocol: domain.NetworkProtocolUDP}]
621 })).Return([]domain.ContainerBackendBind{
622 {ContainerPort: 9000, HostPort: 19000, Protocol: domain.NetworkProtocolTCP},
623 {ContainerPort: 9000, HostPort: 19001, Protocol: domain.NetworkProtocolUDP},
624 }, nil).Once()
625 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
626 if len(claims) != 2 {
627 return false
628 }
629 byProto := map[string]domain.AppListenerReservation{}
630 for _, claim := range claims {
631 if claim.Owner != domain.OwnerGordonBackend || claim.ContainerID != "c-new" || claim.IP != "127.0.0.1" {
632 return false
633 }
634 byProto[claim.Proto] = claim
635 }
636 return byProto["tcp"].Port == 19000 && byProto["udp"].Port == 19001
637 })).Return(nil).Once()
638 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{}}, true, nil)
639 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(a domain.AppActive) bool {
640 svc := a.Services["web"]
641 return svc.BackendBinds[9000] == 19000 && svc.UDPBackendBinds[9000] == 19001
642 })).Return(nil).Once()
643 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil).Once()
644
645 svc := deployment.NewService(
646 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
647 zerowrap.Default(),
648 ).WithProbeDeps(deployment.NewProbeDeps(runtime))
649
650 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
651 require.NoError(t, err)
652 require.NotNil(t, result)
653 assert.Equal(t, "deployed", result.Services["web"].Result)
654 assert.Equal(t, map[int]int{9000: 19000}, result.Services["web"].BackendBinds)
655 assert.Equal(t, map[int]int{9000: 19001}, result.Services["web"].UDPBackendBinds)
656}
657
658// TestDeploy_Mockery_InjectsAppEnvAndSecrets proves the captured revision's
659// app-wide public env reaches every service container alongside that
660// service's own resolved secrets, sorted and deterministic. Public-only
661// and empty-value cases are covered; resolved secret values never enter
662// saved state (SaveActive/SaveOwnership carry no KEY=value payloads).
663func TestDeploy_Mockery_InjectsAppEnvAndSecrets(t *testing.T) {
664 ctx := context.Background()
665 state, runtime, images, secrets := mockDeps(t)
666 svcSpec := webService()
667 svcSpec.HTTP = nil
668 svcSpec.Readiness = domain.AppReadiness{}
669 rev := domain.AppDesiredRevision{
670 Revision: "rev-1", App: "blog",
671 Spec: domain.AppSpec{
672 Name: "blog",
673 Env: map[string]string{"APP_ENV": "production", "EMPTY_OK": ""},
674 Services: []domain.AppService{
675 svcSpec,
676 {
677 Name: "worker", Image: "docker.io/example/worker:1.4.2",
678 StopGrace: 10 * time.Second,
679 Readiness: domain.AppReadiness{Timeout: 30 * time.Second},
680 Secrets: map[string]string{},
681 },
682 },
683 },
684 }
685
686 state.EXPECT().Recover(mock.Anything).Return(nil)
687 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
688 images.EXPECT().ResolveDigest(mock.Anything, rev.Spec.Services[0].Image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil).Once()
689 images.EXPECT().ResolveDigest(mock.Anything, rev.Spec.Services[1].Image).Return("sha256:"+strings.Repeat("b", 64), nil).Once()
690 secrets.EXPECT().GetSecret(mock.Anything, "gordon/apps/app-blog/web/database-url").Return("s3cr3t", nil).Twice()
691 runtime.EXPECT().InspectImageVolumes(mock.Anything, rev.Spec.Services[0].Image).Return(nil, nil).Once()
692 runtime.EXPECT().InspectImageVolumes(mock.Anything, rev.Spec.Services[1].Image).Return(nil, nil).Once()
693 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil).Once()
694 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
695 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
696 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
697 App: "blog", Services: map[string]domain.AppEffectiveService{},
698 }, true, nil)
699 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
700 expectNetworkProvision(runtime, "app-blog", 2)
701
702 var webEnv, workerEnv []string
703 runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool {
704 if len(cfg.Env) != 3 {
705 return false
706 }
707 webEnv = append([]string(nil), cfg.Env...)
708 return true
709 })).Return(&domain.Container{ID: "c-web", Name: "web"}, nil).Once()
710 runtime.EXPECT().CreateContainer(mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool {
711 workerEnv = append([]string(nil), cfg.Env...)
712 return len(cfg.Env) == 2
713 })).Return(&domain.Container{ID: "c-worker", Name: "worker"}, nil).Once()
714 runtime.EXPECT().StartContainer(mock.Anything, mock.Anything).Return(nil)
715 runtime.EXPECT().InspectContainer(mock.Anything, mock.Anything).Return(&domain.Container{Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
716 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
717 App: "blog", Services: map[string]domain.AppEffectiveService{},
718 }, true, nil)
719 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(a domain.AppActive) bool {
720 for _, svc := range a.Services {
721 for _, kv := range []string{svc.Image, svc.Digest, svc.Container} {
722 if kv == "s3cr3t" || kv == "DATABASE_URL=s3cr3t" {
723 return false
724 }
725 }
726 }
727 return true
728 })).Return(nil)
729 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
730
731 svc := deployment.NewService(
732 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
733 zerowrap.Default(),
734 ).WithProbeDeps(deployment.NewProbeDeps(runtime))
735
736 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
737 require.NoError(t, err)
738 require.NotNil(t, result)
739 assert.Equal(t, []string{"APP_ENV=production", "DATABASE_URL=s3cr3t", "EMPTY_OK="}, webEnv)
740 assert.Equal(t, []string{"APP_ENV=production", "EMPTY_OK="}, workerEnv)
741}
742
743// TestStart_RunningContainerRefreshesShiftedBinds proves boot/start bind
744// verification: a running container whose ephemeral loopback bind shifted
745// (native restart while the daemon was away) gets its ACTIVE record
746// updated before the proxy can dial the stale bind. The ACTIVE
747// container is never stopped or removed by re-inspection.
748func TestStart_RunningContainerRefreshesShiftedBinds(t *testing.T) {
749 ctx := context.Background()
750 state, runtime, images, secrets := mockDeps(t)
751 rev := mockRevision()
752
753 state.EXPECT().Recover(mock.Anything).Return(nil)
754 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
755 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
756 active := domain.AppActive{
757 App: "blog",
758 Services: map[string]domain.AppEffectiveService{
759 "web": {
760 EffectiveRevision: "rev-1",
761 Image: rev.Spec.Services[0].Image,
762 Digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
763 Container: "c-old",
764 Spec: rev.Spec.Services[0],
765 BackendBinds: map[int]int{8080: 32770},
766 },
767 },
768 }
769 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
770 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
771 runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(true, nil)
772 // Native restart shifted the ephemeral bind: 32770 -> 32771.
773 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
774 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
775 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
776 return len(claims) == 1 && claims[0].Port == 32771 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-old"
777 })).Return(nil).Once()
778 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
779 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(a domain.AppActive) bool {
780 return a.Services["web"].BackendBinds[8080] == 32771
781 })).Return(nil).Once()
782
783 svc := deployment.NewService(
784 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
785 zerowrap.Default(),
786 ).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
787 func(context.Context, string, string) (int, error) { return 200, nil },
788 func(context.Context, string) error { return nil },
789 ))
790 result, err := svc.Start(ctx, "blog", "")
791 require.NoError(t, err)
792 require.NotNil(t, result)
793 require.Contains(t, result.Services, "web")
794 assert.Equal(t, "deployed", result.Services["web"].Result)
795 assert.Equal(t, map[int]int{8080: 32771}, result.Services["web"].BackendBinds)
796 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
797 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
798}
799
800// TestStart_BindVerificationFailureFailsClosed proves an unverified bind
801// is never served: when re-inspection fails on a running container, the
802// service step fails (fail-closed) without touching the container, and
803// the stale recorded bind is withdrawn from ACTIVE so the next traffic
804// rebuild cannot re-expose a dead or recycled loopback port.
805func TestStart_BindVerificationFailureFailsClosed(t *testing.T) {
806 ctx := context.Background()
807 state, runtime, images, secrets := mockDeps(t)
808 rev := mockRevision()
809
810 state.EXPECT().Recover(mock.Anything).Return(nil)
811 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
812 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
813 active := domain.AppActive{
814 App: "blog",
815 Services: map[string]domain.AppEffectiveService{
816 "web": {
817 EffectiveRevision: "rev-1",
818 Image: rev.Spec.Services[0].Image,
819 Digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
820 Container: "c-old",
821 Spec: rev.Spec.Services[0],
822 BackendBinds: map[int]int{8080: 32770},
823 },
824 },
825 }
826 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
827 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
828 runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(true, nil)
829 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
830 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(nil, assert.AnError).Once()
831
832 traffic := &recordingTraffic{}
833 svc := deployment.NewService(
834 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic},
835 zerowrap.Default(),
836 )
837 result, err := svc.Start(ctx, "blog", "")
838 require.NoError(t, err)
839 require.NotNil(t, result)
840 assert.Equal(t, "failed", result.Services["web"].Result)
841 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
842 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
843 // The stale bind is withdrawn through the canonical boundary
844 // (fail-closed projection); deployment writes no ACTIVE binds itself.
845 assert.Equal(t, []string{"blog/web", "blog/web"}, traffic.withdrawn())
846 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
847}
848
849// TestRestart_MixedServiceRefreshesBothProtocols proves a runtime restart
850// re-inspects TCP and UDP binds together: shifted ephemeral ports on
851// both protocols persist to ACTIVE and surface in the result.
852func TestRestart_MixedServiceRefreshesBothProtocols(t *testing.T) {
853 ctx := context.Background()
854 state, runtime, images, secrets := mockDeps(t)
855 svcSpec := webService()
856 svcSpec.HTTP = nil
857 svcSpec.TCP = []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 9000, Publish: "9000"}}
858 svcSpec.UDP = []domain.AppUDPInterface{{Entrypoint: "udp", Port: 9000, Publish: "9000"}}
859 svcSpec.Readiness = domain.AppReadiness{Type: "none", Timeout: time.Second}
860 svcSpec.Secrets = map[string]string{}
861 active := domain.AppActive{
862 App: "blog",
863 Services: map[string]domain.AppEffectiveService{
864 "web": {
865 EffectiveRevision: "rev-1",
866 Image: svcSpec.Image,
867 Digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
868 Container: "c-old",
869 Spec: svcSpec,
870 BackendBinds: map[int]int{9000: 32770},
871 UDPBackendBinds: map[int]int{9000: 32780},
872 },
873 },
874 }
875 state.EXPECT().Recover(mock.Anything).Return(nil)
876 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
877 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
878 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
879 runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
880 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
881 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return([]domain.ContainerBackendBind{
882 {ContainerPort: 9000, HostPort: 32771, Protocol: domain.NetworkProtocolTCP},
883 {ContainerPort: 9000, HostPort: 32781, Protocol: domain.NetworkProtocolUDP},
884 }, nil).Once()
885 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
886 if len(claims) != 2 {
887 return false
888 }
889 byProto := map[string]int{}
890 for _, claim := range claims {
891 byProto[claim.Proto] = claim.Port
892 }
893 return byProto["tcp"] == 32771 && byProto["udp"] == 32781
894 })).Return(nil).Once()
895 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
896 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(a domain.AppActive) bool {
897 svc := a.Services["web"]
898 return svc.BackendBinds[9000] == 32771 && svc.UDPBackendBinds[9000] == 32781
899 })).Return(nil).Once()
900
901 svc := deployment.NewService(
902 deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
903 zerowrap.Default(),
904 )
905 result, err := svc.Restart(ctx, "blog", "", "")
906 require.NoError(t, err)
907 require.NotNil(t, result)
908 assert.Equal(t, "deployed", result.Services["web"].Result)
909 assert.Equal(t, map[int]int{9000: 32771}, result.Services["web"].BackendBinds)
910 assert.Equal(t, map[int]int{9000: 32781}, result.Services["web"].UDPBackendBinds)
911}