1package deployment
2
3import (
4 "context"
5 "errors"
6 "sort"
7 "time"
8
9 "github.com/bnema/zerowrap"
10
11 "github.com/bnema/gordon/internal/domain"
12)
13
14// stopGraceOrDefault returns the grace a runtime call must use for one
15// container. A zero declared grace means the app default: an app path
16// never hands the runtime an immediate kill.
17func stopGraceOrDefault(grace time.Duration) time.Duration {
18 if grace <= 0 {
19 return domain.AppDefaultStopGrace
20 }
21 return grace
22}
23
24// serviceStopGrace returns the effective stop grace of one service.
25func serviceStopGrace(eff domain.AppEffectiveService) time.Duration {
26 return stopGraceOrDefault(eff.Spec.StopGrace)
27}
28
29// activeStopGrace returns the effective stop grace recorded for one
30// service of an ACTIVE record, or the app default when the record has no
31// such service. Retiring a replaced container uses the grace of the
32// generation being stopped, never the grace of its replacement.
33func activeStopGrace(active domain.AppActive, service string) time.Duration {
34 return stopGraceOrDefault(active.Services[service].Spec.StopGrace)
35}
36
37// retireOptions control one exact-container retirement.
38type retireOptions struct {
39 // Service names the app service for warnings.
40 Service string
41 // Grace is the effective stop grace of the container being retired.
42 Grace time.Duration
43 // Force removes a container that was never published and holds no
44 // committed state, without waiting for a grace period.
45 Force bool
46 // ClearInhibition drops the generation's recovery inhibition once
47 // the container is confirmed gone. It is only safe when nothing can
48 // recreate this generation: an operator stop or a removal. A
49 // replacement keeps its inhibition until the new generation is
50 // published, because recovery must not recreate the replaced writer
51 // from the still-active old revision.
52 ClearInhibition bool
53 // RetryOnLeak keeps the inhibition when the container is gone but its
54 // backend claims could not be released, so a durable retirement marker is
55 // retried (retiring a gone container is idempotent: not-found counts as
56 // gone) instead of leaking the claims untracked. Callers that clear an
57 // inhibition for a container that must simply never be revived (stop,
58 // remove) leave it unset: a lingering marker there would refuse the
59 // next start.
60 RetryOnLeak bool
61}
62
63// containerRetirement is the outcome of one exact-container retirement.
64type containerRetirement struct {
65 // ContainerID is the container the retirement targeted.
66 ContainerID string
67 // Gone is true only when the runtime confirmed the container no
68 // longer exists. Not-found counts as confirmation.
69 Gone bool
70 // Settled is true when the container is gone AND its backend claims were
71 // released. Only a settled retirement may drop a durable marker; a gone
72 // container whose claims leaked is retried, which is idempotent.
73 Settled bool
74 // Warnings are bounded, log-free leftovers for operator action.
75 Warnings []CleanupWarning
76}
77
78// retireContainer is the single retirement path for one exact container:
79// stop it with the effective grace (or force-remove a never-published
80// candidate), confirm the exact ID is gone, and only then release its
81// backend claims and clear its recovery inhibition. A container that is
82// not confirmed gone keeps its claims and its inhibition, so no later
83// generation can dial a recycled loopback port and no recovery pass can
84// revive a half-removed generation.
85func (s *Service) retireContainer(ctx context.Context, app string, opts retireOptions, containerID string) containerRetirement {
86 result := containerRetirement{ContainerID: containerID}
87 if containerID == "" {
88 result.Gone = true
89 result.Settled = true
90 return result
91 }
92 if err := s.stopForRetirement(ctx, opts, containerID); err != nil {
93 result.Warnings = append(result.Warnings, CleanupWarning{
94 Service: opts.Service, Leftover: containerID, Detail: "stop: " + err.Error(),
95 })
96 return result
97 }
98 if err := s.deps.Runtime.RemoveContainer(ctx, containerID, opts.Force); err != nil && !errors.Is(err, domain.ErrContainerNotFound) {
99 result.Warnings = append(result.Warnings, CleanupWarning{
100 Service: opts.Service, Leftover: containerID, Detail: "remove: " + err.Error(),
101 })
102 return result
103 }
104 result.Gone = true
105 released := true
106 if err := s.deps.State.ReleaseBackendBinds(ctx, app, containerID); err != nil {
107 released = false
108 result.Warnings = append(result.Warnings, CleanupWarning{
109 Service: opts.Service, Leftover: containerID, Detail: "release backend claims: " + err.Error(),
110 })
111 }
112 // The app lock is held, so no candidate of this app can be between
113 // secret file write and container create.
114 if err := s.sweepSecretFiles(ctx, app); err != nil {
115 // Fixed detail: the error may name host paths, which must stay
116 // out of journals and API responses.
117 log := zerowrap.FromCtx(ctx)
118 log.Warn().Err(err).Str("app", app).Str("service", opts.Service).Msg("deployment: secret files cleanup failed")
119 result.Warnings = append(result.Warnings, CleanupWarning{
120 Service: opts.Service, Leftover: containerID, Detail: "secret files cleanup failed",
121 })
122 }
123 result.Settled = released
124 if opts.ClearInhibition && (released || !opts.RetryOnLeak) {
125 // Confirmed gone: the marker has nothing left to protect, and a
126 // stale record would only confuse the next recreation. With
127 // RetryOnLeak it also waits for the claims to be released.
128 if err := s.clearRecoveryInhibition(ctx, app, opts.Service, containerID); err != nil {
129 result.Warnings = append(result.Warnings, CleanupWarning{
130 Service: opts.Service, Leftover: containerID, Detail: "clear recovery inhibition: " + err.Error(),
131 })
132 }
133 }
134 return result
135}
136
137// stopForRetirement stops one container. A forced retirement removes
138// without a grace period: only a candidate that was never published and
139// holds no committed state may be retired that way.
140func (s *Service) stopForRetirement(ctx context.Context, opts retireOptions, containerID string) error {
141 if opts.Force {
142 return nil
143 }
144 if err := s.deps.Runtime.StopContainer(ctx, containerID, stopGraceOrDefault(opts.Grace)); err != nil && !errors.Is(err, domain.ErrContainerNotFound) {
145 return err
146 }
147 return nil
148}
149
150// retireCandidate removes a candidate that was never published and
151// releases its backend claims. The candidate holds no committed state, so
152// it is force-removed without a grace period. Cleanup failures are
153// returned as bounded warnings: a candidate that could not be removed
154// must not hide the failure the caller is already reporting.
155func (s *Service) retireCandidate(ctx context.Context, app, service, containerID string) []CleanupWarning {
156 retired := s.retireContainer(ctx, app, retireOptions{Service: service, Force: true}, containerID)
157 return retired.Warnings
158}
159
160// cleanupDetail renders the first bounded warning of a failed retirement
161// for an error message. Journal and log content never travel here.
162func cleanupDetail(retired containerRetirement) string {
163 if len(retired.Warnings) == 0 {
164 return "container not confirmed gone"
165 }
166 return retired.Warnings[0].Detail
167}
168
169// collectCleanupWarnings gathers one operation's per-service leftovers in
170// stable service order. It is the single source of the operation-level
171// warning list.
172func collectCleanupWarnings(results map[string]ServiceResult) []CleanupWarning {
173 names := make([]string, 0, len(results))
174 for name := range results {
175 names = append(names, name)
176 }
177 sort.Strings(names)
178 var warnings []CleanupWarning
179 for _, name := range names {
180 warnings = append(warnings, results[name].CleanupWarnings...)
181 }
182 return warnings
183}
184
185// journalWarnings maps leftovers into the bounded journal shape. The
186// service name travels with each warning, so a leftover is always
187// attributable without parsing its text.
188func journalWarnings(leftovers []CleanupWarning) []domain.AppOperationWarning {
189 var warnings []domain.AppOperationWarning
190 for _, warning := range leftovers {
191 warnings = append(warnings, domain.AppOperationWarning{
192 Service: warning.Service,
193 Leftover: warning.Leftover,
194 Detail: warning.Detail,
195 })
196 }
197 return warnings
198}