1package deployment_test
2
3import (
4 "context"
5 "testing"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/mock"
11 "github.com/stretchr/testify/require"
12
13 "github.com/bnema/gordon/internal/domain"
14 "github.com/bnema/gordon/internal/usecase/deployment"
15)
16
17// graceSpec is a headless service carrying one explicit stop grace.
18func graceSpec(grace time.Duration) domain.AppService {
19 spec := headlessSpec()
20 spec.StopGrace = grace
21 return spec
22}
23
24func activeWithGrace(container string, grace time.Duration) domain.AppActive {
25 return domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
26 "web": {Container: container, EffectiveRevision: "rev-1", Spec: graceSpec(grace)},
27 }}
28}
29
30// TestLifecycleVerbsPassTheEffectiveStopGrace proves every runtime stop
31// uses the effective service grace instead of a fixed adapter timeout.
32func TestLifecycleVerbsPassTheEffectiveStopGrace(t *testing.T) {
33 const grace = 25 * time.Second
34
35 t.Run("stop", func(t *testing.T) {
36 ctx := context.Background()
37 state, runtime, _, _ := mockDeps(t)
38 active := activeWithGrace("c-1", grace)
39 state.EXPECT().Recover(mock.Anything).Return(nil)
40 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
41 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
42 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
43 runtime.EXPECT().StopContainer(mock.Anything, "c-1", grace).Return(nil).Once()
44 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
45 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
46 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
47 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
48
49 svc := deployment.NewService(deployment.Deps{State: state, Runtime: runtime}, zerowrap.Default())
50 _, err := svc.Stop(ctx, "blog", "")
51 require.NoError(t, err)
52 })
53
54 t.Run("restart", func(t *testing.T) {
55 ctx := context.Background()
56 state, runtime, _, _ := mockDeps(t)
57 active := activeWithGrace("c-1", grace)
58 state.EXPECT().Recover(mock.Anything).Return(nil)
59 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
60 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
61 runtime.EXPECT().RestartContainer(mock.Anything, "c-1", grace).Return(nil).Once()
62 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
63 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
64
65 svc := deployment.NewService(deployment.Deps{State: state, Runtime: runtime}, zerowrap.Default())
66 _, err := svc.Restart(ctx, "blog", "web", "")
67 require.NoError(t, err)
68 })
69
70 t.Run("remove", func(t *testing.T) {
71 ctx := context.Background()
72 state, runtime, _, _ := mockDeps(t)
73 active := activeWithGrace("c-1", grace)
74 state.EXPECT().Recover(mock.Anything).Return(nil)
75 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
76 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
77 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
78 runtime.EXPECT().StopContainer(mock.Anything, "c-1", grace).Return(nil).Once()
79 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
80 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
81 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
82 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
83 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
84 state.EXPECT().RetireApp(mock.Anything, "blog").Return(nil).Once()
85 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
86
87 svc := deployment.NewService(deployment.Deps{State: state, Runtime: runtime}, zerowrap.Default())
88 _, err := svc.Remove(ctx, "blog", "")
89 require.NoError(t, err)
90 })
91}
92
93// TestStopWithoutDeclaredGraceUsesTheAppDefault proves an effective spec
94// with no stop_grace still reaches the runtime as the app default instead
95// of an immediate kill.
96func TestStopWithoutDeclaredGraceUsesTheAppDefault(t *testing.T) {
97 ctx := context.Background()
98 state, runtime, _, _ := mockDeps(t)
99 active := activeWithGrace("c-1", 0)
100 state.EXPECT().Recover(mock.Anything).Return(nil)
101 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
102 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
103 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
104 runtime.EXPECT().StopContainer(mock.Anything, "c-1", domain.AppDefaultStopGrace).Return(nil).Once()
105 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
106 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
107 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
108 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
109
110 svc := deployment.NewService(deployment.Deps{State: state, Runtime: runtime}, zerowrap.Default())
111 _, err := svc.Stop(ctx, "blog", "")
112 require.NoError(t, err)
113}
114
115// TestRemove_FailedRetirementKeepsClaimsAndActiveState proves a container
116// that is not confirmed gone keeps its backend claims, keeps its ACTIVE
117// reference, and fails the operation terminally.
118func TestRemove_FailedRetirementKeepsClaimsAndActiveState(t *testing.T) {
119 ctx := context.Background()
120 state, runtime, _, _ := mockDeps(t)
121 active := activeWithGrace("c-1", time.Second)
122 state.EXPECT().Recover(mock.Anything).Return(nil)
123 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
124 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
125 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
126 runtime.EXPECT().StopContainer(mock.Anything, "c-1", time.Second).Return(nil).Once()
127 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(assert.AnError).Once()
128 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
129
130 svc := deployment.NewService(deployment.Deps{State: state, Runtime: runtime}, zerowrap.Default())
131 _, err := svc.Remove(ctx, "blog", "")
132 require.Error(t, err)
133 assert.Contains(t, err.Error(), "not confirmed gone")
134 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
135 state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
136 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
137 state.AssertNotCalled(t, "RetireApp", mock.Anything, mock.Anything)
138}
139
140// TestDeploy_RetirementFailureBlocksReplacement proves the superseded
141// generation must be confirmed gone before a replacement starts: a failed
142// retirement aborts the service without creating a candidate, and the
143// failure is journaled instead of being downgraded to a warning.
144func TestDeploy_RetirementFailureBlocksReplacement(t *testing.T) {
145 ctx := context.Background()
146 state, runtime, images, secrets := mockDeps(t)
147 rev := statefulMockRevision()
148
149 oldActive := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
150 "web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", Spec: graceSpec(time.Second)},
151 }}
152
153 state.EXPECT().Recover(mock.Anything).Return(nil)
154 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
155 images.EXPECT().ResolveDigest(mock.Anything, rev.Spec.Services[0].Image).Return(
156 "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil).Once()
157 secrets.EXPECT().GetSecret(mock.Anything, "gordon/apps/app-blog/web/database-url").Return("x", nil)
158 runtime.EXPECT().InspectImageVolumes(mock.Anything, rev.Spec.Services[0].Image).Return(nil, nil).Once()
159 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil).Once()
160 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
161 state.EXPECT().LoadActive(mock.Anything, "blog").Return(oldActive, true, nil)
162 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
163 runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil)
164 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
165 // The superseded container is retired with its own effective grace, and
166 // its removal fails: the replacement must not start.
167 runtime.EXPECT().StopContainer(mock.Anything, "c-old", time.Second).Return(nil).Once()
168 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
169
170 var saved []domain.AppOperation
171 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).RunAndReturn(
172 func(_ context.Context, op domain.AppOperation) error {
173 saved = append(saved, op)
174 return nil
175 })
176
177 svc := deployment.NewService(deployment.Deps{
178 State: state, Runtime: runtime, Images: images, Secrets: secrets,
179 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
180 func(context.Context, string, string) (int, error) { return 200, nil },
181 func(context.Context, string) error { return nil },
182 ))
183
184 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
185 require.Error(t, err)
186 require.NotNil(t, result)
187 assert.Equal(t, "failed", result.Services["web"].Result)
188 assert.Contains(t, result.Services["web"].Error, "retire superseded container",
189 "the unconfirmed retirement is the reported failure")
190
191 // No workload is created, started, or published while the superseded
192 // generation is not confirmed gone, and no claim or inhibition of that
193 // generation is released.
194 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
195 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
196 runtime.AssertNotCalled(t, "ListNetworks", mock.Anything)
197 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
198 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
199 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
200 state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
201
202 require.NotEmpty(t, saved)
203 last := saved[len(saved)-1]
204 require.Len(t, last.Steps, 2)
205 assert.Equal(t, "preflight", last.Steps[0].ID)
206 assert.Equal(t, domain.AppStepSucceeded, last.Steps[0].State)
207 assert.Equal(t, "service.web.replace", last.Steps[1].ID)
208 assert.Equal(t, domain.AppStepFailed, last.Steps[1].State)
209 assert.Equal(t, domain.AppOutcomeFailed, last.Outcome)
210}