1package deployment_test
2
3import (
4 "context"
5 "testing"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/mock"
11 "github.com/stretchr/testify/require"
12
13 "github.com/bnema/gordon/internal/adapters/out/appstate"
14 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
15 "github.com/bnema/gordon/internal/domain"
16 "github.com/bnema/gordon/internal/usecase/deployment"
17)
18
19// newTestStore opens a real app-state store in a temporary directory.
20func newTestStore(t *testing.T) *appstate.Store {
21 t.Helper()
22 store, err := appstate.NewStore(t.TempDir(), zerowrap.Default())
23 require.NoError(t, err)
24 t.Cleanup(func() { require.NoError(t, store.Close()) })
25 return store
26}
27
28// TestRecoveryInhibition_RefusesInhibitedGenerationAfterReboot proves the
29// durable inhibition contract end to end: an old volume-owning container
30// that a replacement may already have superseded is never started again,
31// even by a completely reconstructed engine reading a reopened store.
32// Unrelated generations of the same app stay recoverable.
33func TestRecoveryInhibition_RefusesInhibitedGenerationAfterReboot(t *testing.T) {
34 ctx := context.Background()
35 dir := t.TempDir()
36
37 store, err := appstate.NewStore(dir, zerowrap.Default())
38 require.NoError(t, err)
39 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
40 require.NoError(t, store.SaveActive(ctx, domain.AppActive{
41 App: "blog",
42 Services: map[string]domain.AppEffectiveService{
43 "web": {
44 Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec(),
45 BackendBinds: map[int]int{8080: 32770},
46 },
47 "worker": {
48 Container: "c-worker", EffectiveRevision: "rev-1", Spec: headlessSpec(),
49 },
50 },
51 }))
52 require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
53 App: "blog", Service: "web", ContainerID: "c-old",
54 Reason: domain.AppInhibitReplacementPending, Operation: "op-1",
55 }))
56 require.NoError(t, store.Close())
57
58 // Reboot: a fresh store handle and a fresh engine, no in-memory state.
59 reopened, err := appstate.NewStore(dir, zerowrap.Default())
60 require.NoError(t, err)
61 t.Cleanup(func() { require.NoError(t, reopened.Close()) })
62 inhibitions, err := reopened.LoadRecoveryInhibitions(ctx, "blog")
63 require.NoError(t, err)
64 require.Len(t, inhibitions, 1)
65 assert.Equal(t, "c-old", inhibitions[0].ContainerID)
66
67 runtime := outmocks.NewMockContainerRuntime(t)
68 runtime.EXPECT().InspectContainer(mock.Anything, "c-worker").
69 Return(&domain.Container{ID: "c-worker", Status: "running", StartedAt: time.Now().UTC()}, nil).Twice()
70 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-worker").Return("", false, nil).Once()
71
72 svc := deployment.NewService(deployment.Deps{
73 State: reopened, Runtime: runtime, Traffic: &recordingTraffic{},
74 }, zerowrap.Default())
75
76 err = svc.ReconcileRunning(ctx)
77 require.Error(t, err)
78 assert.Contains(t, err.Error(), "recovery inhibited")
79 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, "c-old")
80 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-old", mock.Anything)
81 runtime.AssertCalled(t, "InspectContainer", mock.Anything, "c-worker")
82 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
83 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
84}
85
86// seedDesired writes one accepted desired revision through the real
87// staged/committed/materialized apply protocol.
88func seedDesired(t *testing.T, ctx context.Context, store *appstate.Store, rev domain.AppDesiredRevision) {
89 t.Helper()
90 intentID := "intent-1"
91 require.NoError(t, store.StageApply(ctx, domain.AppApplyIntent{
92 Intent: intentID, App: rev.App, Revision: rev.Revision,
93 SourceSHA256: "0f1e2d", Spec: rev.Spec, CreatedAt: time.Now().UTC(),
94 }))
95 require.NoError(t, store.CommitApply(ctx, rev.App, intentID))
96 require.NoError(t, store.MaterializeApply(ctx, rev.App, intentID))
97}
98
99// TestVolumeReplacementFailureLeavesOldGenerationInhibited proves the
100// marker is written BEFORE a volume-owning replacement can write and is
101// not cleared when that replacement fails: the old generation must never
102// be restarted on top of possibly-newer data.
103func TestVolumeReplacementFailureLeavesOldGenerationInhibited(t *testing.T) {
104 ctx := context.Background()
105 store := newTestStore(t)
106 runtime := outmocks.NewMockContainerRuntime(t)
107 images := outmocks.NewMockImageResolver(t)
108 secrets := outmocks.NewMockSecretProvider(t)
109
110 spec := domain.AppService{
111 Name: "web",
112 Image: "registry.example.com/blog/web:1.4.2",
113 Volumes: []domain.AppVolume{{Name: "data", Path: "/data"}},
114 Secrets: map[string]string{},
115 Readiness: domain.AppReadiness{
116 Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: 50 * time.Millisecond,
117 },
118 }
119 rev := testRevision("blog", spec)
120 rev.Spec.Env = map[string]string{}
121
122 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
123 require.NoError(t, store.SaveActive(ctx, domain.AppActive{
124 App: "blog",
125 Services: map[string]domain.AppEffectiveService{
126 "web": {
127 Container: "c-old", EffectiveRevision: "rev-0", Spec: headlessSpec(),
128 BackendBinds: map[int]int{9000: 19000},
129 },
130 },
131 }))
132 seedDesired(t, ctx, store, rev)
133
134 images.EXPECT().ResolveDigest(mock.Anything, spec.Image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil).Once()
135 runtime.EXPECT().InspectImageVolumes(mock.Anything, spec.Image).Return(nil, nil).Once()
136 runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil).Once()
137 // The superseded writer cannot be removed: the replacement must abort
138 // rather than risk overlapping writers, and the inhibition written
139 // before the stop attempt stays in place.
140 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
141 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
142
143 svc := deployment.NewService(deployment.Deps{
144 State: store, Runtime: runtime, Images: images, Secrets: secrets,
145 ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
146 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
147 func(context.Context, string, string) (int, error) { return 500, nil },
148 func(context.Context, string) error { return assert.AnError },
149 ))
150
151 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
152 require.Error(t, err)
153
154 inhibitions, err := store.LoadRecoveryInhibitions(ctx, "blog")
155 require.NoError(t, err)
156 require.Len(t, inhibitions, 1)
157 assert.Equal(t, "c-old", inhibitions[0].ContainerID)
158 assert.Equal(t, domain.AppInhibitReplacementPending, inhibitions[0].Reason)
159
160 // A later boot must refuse to revive the inhibited generation, and a
161 // rebuilt engine must reach the same conclusion.
162 runtime2 := outmocks.NewMockContainerRuntime(t)
163 rebooted := deployment.NewService(deployment.Deps{
164 State: store, Runtime: runtime2, Images: images, Secrets: secrets,
165 }, zerowrap.Default())
166 bootErr := rebooted.ReconcileBoot(ctx)
167 require.Error(t, bootErr)
168 assert.Contains(t, bootErr.Error(), "recovery inhibited")
169 runtime2.AssertNotCalled(t, "StartContainer", mock.Anything, "c-old")
170 runtime2.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-old", mock.Anything)
171 runtime2.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
172}
173
174// TestRemove_InhibitsBeforeRuntimeWithdrawal proves remove is durably
175// fail-closed: stopped intent and an inhibition per removed container are
176// persisted before any runtime effect, and volumes are never deleted.
177func TestRemove_InhibitsBeforeRuntimeWithdrawal(t *testing.T) {
178 ctx := context.Background()
179 state, runtime, images, secrets := mockDeps(t)
180
181 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
182 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
183 }}
184 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
185 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
186
187 var order []string
188 state.EXPECT().SaveIntent(mock.Anything, mock.MatchedBy(func(intent domain.AppStopIntent) bool {
189 return intent.Stopped
190 })).RunAndReturn(func(context.Context, domain.AppStopIntent) error {
191 order = append(order, "intent-stopped")
192 return nil
193 }).Once()
194 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.MatchedBy(func(inhibition domain.AppRecoveryInhibition) bool {
195 return inhibition.App == "blog" && inhibition.Service == "web" && inhibition.ContainerID == "c-1"
196 })).RunAndReturn(func(context.Context, domain.AppRecoveryInhibition) error {
197 order = append(order, "inhibited")
198 return nil
199 }).Once()
200 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).RunAndReturn(func(context.Context, string, time.Duration) error {
201 order = append(order, "stopped")
202 return nil
203 }).Once()
204 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).RunAndReturn(func(context.Context, string, bool) error {
205 order = append(order, "removed")
206 return nil
207 }).Once()
208 // Confirmed disappearance releases the container's backend claims and
209 // clears its recovery inhibition before the incarnation is retired.
210 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
211 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
212 // Ownership is read while it is still live; the name-only record owns
213 // no network, so nothing is reclaimed.
214 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
215 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
216 // The incarnation is retired atomically after the workload is gone.
217 state.EXPECT().RetireApp(mock.Anything, "blog").RunAndReturn(func(context.Context, string) error {
218 order = append(order, "retired")
219 return nil
220 }).Once()
221 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
222
223 svc := deployment.NewService(deployment.Deps{
224 State: state, Runtime: runtime, Images: images, Secrets: secrets,
225 }, zerowrap.Default())
226 result, err := svc.Remove(ctx, "blog", "")
227 require.NoError(t, err)
228 require.NotNil(t, result)
229 assert.Equal(t, domain.AppServiceRemoved, result.Services["web"].Result)
230 assert.Equal(t, []string{"intent-stopped", "inhibited", "stopped", "removed", "retired"}, order)
231 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
232}
233
234// TestBootStoppedConvergenceStopsRevivedContainer proves durable stopped
235// intent wins over native restart policy: a container revived while the
236// daemon was away is stopped again, without removal, and is never started.
237func TestBootStoppedConvergenceStopsRevivedContainer(t *testing.T) {
238 ctx := context.Background()
239 state, runtime, images, secrets := mockDeps(t)
240
241 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
242 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
243 }}
244 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
245 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
246 state.EXPECT().LoadIntent(mock.Anything, "blog").
247 Return(domain.AppStopIntent{App: "blog", Stopped: true}, nil).Once()
248 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
249 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
250 Return(&domain.Container{ID: "c-1", Status: "running"}, nil).Once()
251 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(nil).Once()
252
253 svc := deployment.NewService(deployment.Deps{
254 State: state, Runtime: runtime, Images: images, Secrets: secrets,
255 }, zerowrap.Default())
256 require.NoError(t, svc.ReconcileBoot(ctx))
257 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
258 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
259 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
260}
261
262// TestBootRefusesInhibitedGenerationBeforeAnyRuntimeEffect proves boot
263// recovery checks the durable marker before touching the workload.
264func TestBootRefusesInhibitedGenerationBeforeAnyRuntimeEffect(t *testing.T) {
265 ctx := context.Background()
266 state, runtime, images, secrets := mockDeps(t)
267
268 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
269 "web": {Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec()},
270 }}
271 state.EXPECT().Recover(mock.Anything).Return(nil)
272 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
273 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
274 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
275 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
276 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").
277 Return([]domain.AppRecoveryInhibition{{App: "blog", Service: "web", ContainerID: "c-old"}}, nil).Once()
278 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
279
280 svc := deployment.NewService(deployment.Deps{
281 State: state, Runtime: runtime, Images: images, Secrets: secrets,
282 }, zerowrap.Default())
283 err := svc.ReconcileBoot(ctx)
284 require.Error(t, err)
285 assert.Contains(t, err.Error(), "recovery inhibited")
286 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
287 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
288 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
289}
290
291// TestWorkloadMutationsSerializePerApp proves a periodic pass cannot
292// interleave with an in-flight user mutation of the same app: the pass
293// skips the busy app instead of waiting for it.
294func TestWorkloadMutationsSerializePerApp(t *testing.T) {
295 ctx := context.Background()
296 state, runtime, images, secrets := mockDeps(t)
297
298 entered := make(chan struct{})
299 release := make(chan struct{})
300 state.EXPECT().Recover(mock.Anything).RunAndReturn(func(context.Context) error {
301 close(entered)
302 <-release
303 return assert.AnError
304 }).Once()
305 // Only the app list may be read: the pass must not touch intent or
306 // ACTIVE while the mutation holds the app lock.
307 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
308
309 svc := deployment.NewService(deployment.Deps{
310 State: state, Runtime: runtime, Images: images, Secrets: secrets,
311 }, zerowrap.Default())
312
313 done := make(chan error, 1)
314 go func() {
315 _, err := svc.Stop(ctx, "blog", "")
316 done <- err
317 }()
318 <-entered
319
320 // The mutation holds the app lock. The periodic pass must skip the
321 // app instead of waiting: LoadIntent has no expectation, so reaching
322 // it would fail the test.
323 require.NoError(t, svc.ReconcileRunning(ctx))
324 state.AssertNotCalled(t, "LoadIntent", mock.Anything, mock.Anything)
325
326 close(release)
327 require.Error(t, <-done)
328}
329
330// TestRestart_RefusesInhibitedGeneration proves an explicit operator
331// restart cannot revive a generation whose recovery is durably inhibited:
332// a replacement may already have written to the volume it still owns.
333func TestRestart_RefusesInhibitedGeneration(t *testing.T) {
334 ctx := context.Background()
335 state, runtime, images, secrets := mockDeps(t)
336
337 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
338 "web": {Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec()},
339 }}
340 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
341 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
342 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return([]domain.AppRecoveryInhibition{{
343 App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitReplacementPending,
344 }}, nil).Once()
345 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
346
347 svc := deployment.NewService(deployment.Deps{
348 State: state, Runtime: runtime, Images: images, Secrets: secrets,
349 }, zerowrap.Default())
350 result, err := svc.Restart(ctx, "blog", "", "")
351 require.ErrorIs(t, err, domain.ErrAppStateConflict)
352 require.NotNil(t, result)
353 assert.Equal(t, "failed", result.Services["web"].Result)
354 assert.Contains(t, result.Services["web"].Error, "recovery inhibited")
355 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
356}
357
358// TestRemove_KeepsStateWhenAContainerCannotBeRemoved proves remove never
359// loses track of a survivor: ACTIVE, stopped intent, and the inhibition
360// stay in place so the periodic pass keeps converging it.
361func TestRemove_KeepsStateWhenAContainerCannotBeRemoved(t *testing.T) {
362 ctx := context.Background()
363 state, runtime, images, secrets := mockDeps(t)
364
365 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
366 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
367 }}
368 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
369 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
370 state.EXPECT().SaveIntent(mock.Anything, mock.MatchedBy(func(intent domain.AppStopIntent) bool {
371 return intent.Stopped
372 })).Return(nil).Once()
373 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
374 // The claim journal is written before the runtime effects and is
375 // rewritten terminally when the removal cannot complete: a repeat of
376 // the same key replays the failure instead of reading a stuck claim.
377 state.EXPECT().SaveOperation(mock.Anything, mock.MatchedBy(func(op domain.AppOperation) bool {
378 return op.Op != "" && op.Kind == "remove" && !op.Terminal()
379 })).Return(nil).Once()
380 state.EXPECT().SaveOperation(mock.Anything, mock.MatchedBy(func(op domain.AppOperation) bool {
381 return op.Op != "" && op.Outcome == domain.AppOutcomeFailed && op.Terminal()
382 })).Return(nil).Once()
383 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(assert.AnError).Once()
384
385 svc := deployment.NewService(deployment.Deps{
386 State: state, Runtime: runtime, Images: images, Secrets: secrets,
387 }, zerowrap.Default())
388 _, err := svc.Remove(ctx, "blog", "")
389 require.Error(t, err)
390 assert.Contains(t, err.Error(), "not confirmed gone")
391 // ACTIVE and the stopped intent were never discarded: the monitor can
392 // still converge the survivor. The survivor's claims and inhibition
393 // stay in place too: only a confirmed disappearance releases them.
394 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
395 state.AssertNotCalled(t, "RetireApp", mock.Anything, mock.Anything)
396 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
397 state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
398 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
399}
400
401// TestRemovalOfMissingContainerIsIdempotent proves a container the
402// runtime already dropped is a successful removal, not a failure.
403func TestRemovalOfMissingContainerIsIdempotent(t *testing.T) {
404 ctx := context.Background()
405 state, runtime, images, secrets := mockDeps(t)
406
407 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
408 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
409 }}
410 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
411 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
412 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
413 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
414 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
415 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(domain.ErrContainerNotFound).Once()
416 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(domain.ErrContainerNotFound).Once()
417 // Not-found is confirmation: the stale claims and the inhibition of
418 // the disappeared container are released.
419 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
420 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
421 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
422 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
423 state.EXPECT().RetireApp(mock.Anything, "blog").Return(nil).Once()
424
425 svc := deployment.NewService(deployment.Deps{
426 State: state, Runtime: runtime, Images: images, Secrets: secrets,
427 }, zerowrap.Default())
428 result, err := svc.Remove(ctx, "blog", "")
429 require.NoError(t, err)
430 require.NotNil(t, result)
431}