package deployment import ( "context" "errors" "sort" "time" "github.com/bnema/zerowrap" "github.com/bnema/gordon/internal/domain" ) // stopGraceOrDefault returns the grace a runtime call must use for one // container. A zero declared grace means the app default: an app path // never hands the runtime an immediate kill. func stopGraceOrDefault(grace time.Duration) time.Duration { if grace <= 0 { return domain.AppDefaultStopGrace } return grace } // serviceStopGrace returns the effective stop grace of one service. func serviceStopGrace(eff domain.AppEffectiveService) time.Duration { return stopGraceOrDefault(eff.Spec.StopGrace) } // activeStopGrace returns the effective stop grace recorded for one // service of an ACTIVE record, or the app default when the record has no // such service. Retiring a replaced container uses the grace of the // generation being stopped, never the grace of its replacement. func activeStopGrace(active domain.AppActive, service string) time.Duration { return stopGraceOrDefault(active.Services[service].Spec.StopGrace) } // retireOptions control one exact-container retirement. type retireOptions struct { // Service names the app service for warnings. Service string // Grace is the effective stop grace of the container being retired. Grace time.Duration // Force removes a container that was never published and holds no // committed state, without waiting for a grace period. Force bool // ClearInhibition drops the generation's recovery inhibition once // the container is confirmed gone. It is only safe when nothing can // recreate this generation: an operator stop or a removal. A // replacement keeps its inhibition until the new generation is // published, because recovery must not recreate the replaced writer // from the still-active old revision. ClearInhibition bool // RetryOnLeak keeps the inhibition when the container is gone but its // backend claims could not be released, so a durable retirement marker is // retried (retiring a gone container is idempotent: not-found counts as // gone) instead of leaking the claims untracked. Callers that clear an // inhibition for a container that must simply never be revived (stop, // remove) leave it unset: a lingering marker there would refuse the // next start. RetryOnLeak bool } // containerRetirement is the outcome of one exact-container retirement. type containerRetirement struct { // ContainerID is the container the retirement targeted. ContainerID string // Gone is true only when the runtime confirmed the container no // longer exists. Not-found counts as confirmation. Gone bool // Settled is true when the container is gone AND its backend claims were // released. Only a settled retirement may drop a durable marker; a gone // container whose claims leaked is retried, which is idempotent. Settled bool // Warnings are bounded, log-free leftovers for operator action. Warnings []CleanupWarning } // retireContainer is the single retirement path for one exact container: // stop it with the effective grace (or force-remove a never-published // candidate), confirm the exact ID is gone, and only then release its // backend claims and clear its recovery inhibition. A container that is // not confirmed gone keeps its claims and its inhibition, so no later // generation can dial a recycled loopback port and no recovery pass can // revive a half-removed generation. func (s *Service) retireContainer(ctx context.Context, app string, opts retireOptions, containerID string) containerRetirement { result := containerRetirement{ContainerID: containerID} if containerID == "" { result.Gone = true result.Settled = true return result } if err := s.stopForRetirement(ctx, opts, containerID); err != nil { result.Warnings = append(result.Warnings, CleanupWarning{ Service: opts.Service, Leftover: containerID, Detail: "stop: " + err.Error(), }) return result } if err := s.deps.Runtime.RemoveContainer(ctx, containerID, opts.Force); err != nil && !errors.Is(err, domain.ErrContainerNotFound) { result.Warnings = append(result.Warnings, CleanupWarning{ Service: opts.Service, Leftover: containerID, Detail: "remove: " + err.Error(), }) return result } result.Gone = true released := true if err := s.deps.State.ReleaseBackendBinds(ctx, app, containerID); err != nil { released = false result.Warnings = append(result.Warnings, CleanupWarning{ Service: opts.Service, Leftover: containerID, Detail: "release backend claims: " + err.Error(), }) } // The app lock is held, so no candidate of this app can be between // secret file write and container create. if err := s.sweepSecretFiles(ctx, app); err != nil { // Fixed detail: the error may name host paths, which must stay // out of journals and API responses. log := zerowrap.FromCtx(ctx) log.Warn().Err(err).Str("app", app).Str("service", opts.Service).Msg("deployment: secret files cleanup failed") result.Warnings = append(result.Warnings, CleanupWarning{ Service: opts.Service, Leftover: containerID, Detail: "secret files cleanup failed", }) } result.Settled = released if opts.ClearInhibition && (released || !opts.RetryOnLeak) { // Confirmed gone: the marker has nothing left to protect, and a // stale record would only confuse the next recreation. With // RetryOnLeak it also waits for the claims to be released. if err := s.clearRecoveryInhibition(ctx, app, opts.Service, containerID); err != nil { result.Warnings = append(result.Warnings, CleanupWarning{ Service: opts.Service, Leftover: containerID, Detail: "clear recovery inhibition: " + err.Error(), }) } } return result } // stopForRetirement stops one container. A forced retirement removes // without a grace period: only a candidate that was never published and // holds no committed state may be retired that way. func (s *Service) stopForRetirement(ctx context.Context, opts retireOptions, containerID string) error { if opts.Force { return nil } if err := s.deps.Runtime.StopContainer(ctx, containerID, stopGraceOrDefault(opts.Grace)); err != nil && !errors.Is(err, domain.ErrContainerNotFound) { return err } return nil } // retireCandidate removes a candidate that was never published and // releases its backend claims. The candidate holds no committed state, so // it is force-removed without a grace period. Cleanup failures are // returned as bounded warnings: a candidate that could not be removed // must not hide the failure the caller is already reporting. func (s *Service) retireCandidate(ctx context.Context, app, service, containerID string) []CleanupWarning { retired := s.retireContainer(ctx, app, retireOptions{Service: service, Force: true}, containerID) return retired.Warnings } // cleanupDetail renders the first bounded warning of a failed retirement // for an error message. Journal and log content never travel here. func cleanupDetail(retired containerRetirement) string { if len(retired.Warnings) == 0 { return "container not confirmed gone" } return retired.Warnings[0].Detail } // collectCleanupWarnings gathers one operation's per-service leftovers in // stable service order. It is the single source of the operation-level // warning list. func collectCleanupWarnings(results map[string]ServiceResult) []CleanupWarning { names := make([]string, 0, len(results)) for name := range results { names = append(names, name) } sort.Strings(names) var warnings []CleanupWarning for _, name := range names { warnings = append(warnings, results[name].CleanupWarnings...) } return warnings } // journalWarnings maps leftovers into the bounded journal shape. The // service name travels with each warning, so a leftover is always // attributable without parsing its text. func journalWarnings(leftovers []CleanupWarning) []domain.AppOperationWarning { var warnings []domain.AppOperationWarning for _, warning := range leftovers { warnings = append(warnings, domain.AppOperationWarning{ Service: warning.Service, Leftover: warning.Leftover, Detail: warning.Detail, }) } return warnings }