1package deployment
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "maps"
8 "sort"
9 "strings"
10 "time"
11
12 "github.com/bnema/zerowrap"
13
14 "github.com/bnema/gordon/internal/domain"
15)
16
17// LifecycleResult carries the terminal outcome of a lifecycle verb.
18type LifecycleResult struct {
19 Op string
20 App string
21 Verb string
22 Outcome string
23 Services map[string]ServiceResult
24 // Warnings are operation-level leftovers that belong to no single
25 // service, such as a private network that could not be verified or
26 // removed.
27 Warnings []CleanupWarning
28}
29
30// Stop persists the durable stopped intent first, then stops and removes
31// exact active containers by ID. Volumes and secrets are retained.
32func (s *Service) Stop(ctx context.Context, app, opID string) (*LifecycleResult, error) {
33 release, err := s.acquireAppContext(ctx, app)
34 if err != nil {
35 return nil, err
36 }
37 defer release()
38 return s.stopLocked(ctx, app, opID)
39}
40
41func (s *Service) stopLocked(ctx context.Context, app, opID string) (*LifecycleResult, error) {
42 ctx = lifecycleCtx(ctx, "Stop", app)
43 log := zerowrap.FromCtx(ctx)
44 if err := s.deps.State.Recover(ctx); err != nil {
45 return nil, fmt.Errorf("deployment: recover before stop: %w", err)
46 }
47 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
48 return nil, err
49 }
50 op := domain.AppOperation{
51 Kind: "stop", App: app,
52 StartedAt: time.Now().UTC(),
53 Request: domain.AppOperationRequestFor("stop", app, "", ""),
54 }
55 op, owned, err := s.claimOperation(ctx, opID, op, []domain.AppOperationStep{{ID: "intent.stopped", State: domain.AppStepPending}})
56 if err != nil {
57 return nil, err
58 }
59 if !owned {
60 return replayedLifecycleResult(op), replayError(op)
61 }
62 if err := s.deps.State.SaveIntent(ctx, domain.AppStopIntent{
63 App: app, Stopped: true, UpdatedBy: op.Op, UpdatedAt: time.Now().UTC(),
64 }); err != nil {
65 intentErr := fmt.Errorf("deployment: persist stopped intent: %w", err)
66 s.failOperation(ctx, &op, intentErr, "error")
67 return nil, intentErr
68 }
69 op.Steps[0].State = domain.AppStepSucceeded
70 active, _, err := s.deps.State.LoadActive(ctx, app)
71 if err != nil {
72 activeErr := fmt.Errorf("deployment: load active: %w", err)
73 s.failOperation(ctx, &op, activeErr, "error")
74 return nil, activeErr
75 }
76 result := &LifecycleResult{Op: op.Op, App: app, Verb: "stop", Services: map[string]ServiceResult{}}
77 var failures []string
78 for _, name := range sortedServiceNames(active) {
79 container := active.Services[name].Container
80 step, warnings, err := s.stopService(ctx, app, name, active.Services[name])
81 op.Steps = append(op.Steps, step)
82 if err != nil {
83 result.Services[name] = ServiceResult{Result: "failed", Before: container, Error: err.Error(), CleanupWarnings: warnings}
84 failures = append(failures, name)
85 continue
86 }
87 result.Services[name] = ServiceResult{Result: "deployed", Before: container, After: "", CleanupWarnings: warnings}
88 }
89 // A superseded predecessor an overlap replacement left running is not in
90 // ACTIVE, so the loop above never saw it: a stopped app must not keep it
91 // running. One that cannot be retired keeps its marker and fails the stop.
92 pendingFailed, pendingErr := s.retirePendingForStop(ctx, app, result)
93 failures = append(failures, pendingFailed...)
94 op.Outcome = ComputeOutcome(result.Services)
95 if pendingErr != nil {
96 // An app-level cleanup failure belongs to no service, so the
97 // per-service results alone would journal a success and a replay of
98 // this key would read it. It is a failed step and a failed outcome.
99 op.Steps = append(op.Steps, domain.AppOperationStep{
100 ID: "pending.retire", State: domain.AppStepFailed, Error: pendingErr.Error(),
101 })
102 op.Outcome = domain.AppOutcomeFailed
103 }
104 op.Warnings = journalWarnings(append(collectCleanupWarnings(result.Services), result.Warnings...))
105 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
106 log.Warn().Err(err).Msg("deployment: failed to record stop outcome")
107 }
108 if err := s.refreshTraffic(ctx, app); err != nil {
109 s.failTrafficPublication(ctx, &op, err)
110 return result, err
111 }
112 if len(failures) > 0 {
113 return result, fmt.Errorf("deployment: stop failed for %s: %w", strings.Join(failures, ", "), domain.ErrAppStateConflict)
114 }
115 return result, nil
116}
117
118// stopService withdraws one service, then retires its exact container
119// with the effective grace. A withdrawal or runtime failure is reported,
120// never treated as a successful stop. The confirmed disappearance
121// releases the container's backend claims and its recovery inhibition:
122// a stopped app is never revived by recovery.
123func (s *Service) stopService(ctx context.Context, app, name string, eff domain.AppEffectiveService) (domain.AppOperationStep, []CleanupWarning, error) {
124 container := eff.Container
125 step := domain.AppOperationStep{ID: "service." + name + ".stop", State: domain.AppStepPending, Before: container}
126 fail := func(err error) (domain.AppOperationStep, []CleanupWarning, error) {
127 step.State = domain.AppStepFailed
128 step.Error = err.Error()
129 return step, nil, err
130 }
131 if err := s.withdrawForRecovery(ctx, app, name); err != nil {
132 return fail(err)
133 }
134 if container != "" {
135 retired := s.retireContainer(ctx, app, retireOptions{
136 Service: name, Grace: serviceStopGrace(eff), ClearInhibition: true,
137 }, container)
138 if !retired.Gone {
139 return fail(fmt.Errorf("deployment: stop %s/%s container %s: %s", app, name, container, cleanupDetail(retired)))
140 }
141 step.After = container
142 step.State = domain.AppStepSucceeded
143 return step, retired.Warnings, nil
144 }
145 step.State = domain.AppStepSucceeded
146 return step, nil, nil
147}
148
149// Start clears the stopped intent and ensures running from active records.
150// It starts missing/stopped instances without duplicating running ones
151// and never activates pending desired revisions.
152func (s *Service) Start(ctx context.Context, app, opID string) (*LifecycleResult, error) {
153 release, err := s.acquireAppContext(ctx, app)
154 if err != nil {
155 return nil, err
156 }
157 defer release()
158 return s.startLocked(ctx, app, opID)
159}
160
161func (s *Service) startLocked(ctx context.Context, app, opID string) (*LifecycleResult, error) {
162 ctx = lifecycleCtx(ctx, "Start", app)
163 log := zerowrap.FromCtx(ctx)
164 if err := s.deps.State.Recover(ctx); err != nil {
165 return nil, fmt.Errorf("deployment: recover before start: %w", err)
166 }
167 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
168 return nil, err
169 }
170 op := domain.AppOperation{
171 Kind: "start", App: app,
172 StartedAt: time.Now().UTC(),
173 Request: domain.AppOperationRequestFor("start", app, "", ""),
174 }
175 op, owned, err := s.claimOperation(ctx, opID, op, []domain.AppOperationStep{{ID: "intent.running", State: domain.AppStepPending}})
176 if err != nil {
177 return nil, err
178 }
179 if !owned {
180 return replayedLifecycleResult(op), replayError(op)
181 }
182 if err := s.deps.State.SaveIntent(ctx, domain.AppStopIntent{
183 App: app, Stopped: false, UpdatedBy: op.Op, UpdatedAt: time.Now().UTC(),
184 }); err != nil {
185 intentErr := fmt.Errorf("deployment: clear stopped intent: %w", err)
186 s.failOperation(ctx, &op, intentErr, "error")
187 return nil, intentErr
188 }
189 op.Steps[0].State = domain.AppStepSucceeded
190 active, _, err := s.deps.State.LoadActive(ctx, app)
191 if err != nil {
192 activeErr := fmt.Errorf("deployment: load active: %w", err)
193 s.failOperation(ctx, &op, activeErr, "error")
194 return nil, activeErr
195 }
196 result := &LifecycleResult{Op: op.Op, App: app, Verb: "start", Services: map[string]ServiceResult{}}
197 for _, name := range sortedServiceNames(active) {
198 eff := active.Services[name]
199 // The step is journaled before the runtime work, so a candidate
200 // created by a redeploy is durably recorded as soon as it exists.
201 op.Steps = append(op.Steps, domain.AppOperationStep{
202 ID: "service." + name + ".start", State: domain.AppStepPending, Before: eff.Container, Service: name,
203 })
204 s.ensureServiceRunning(ctx, app, opID, name, eff, &op, len(op.Steps)-1, result)
205 }
206 op.Outcome = ComputeOutcome(result.Services)
207 op.Warnings = journalWarnings(collectCleanupWarnings(result.Services))
208 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
209 log.Warn().Err(err).Msg("deployment: failed to record start outcome")
210 }
211 if err := s.refreshTraffic(ctx, app); err != nil {
212 s.failTrafficPublication(ctx, &op, err)
213 return result, err
214 }
215 // Routing accepted the verified generations: record their executions.
216 for name, svcResult := range result.Services {
217 s.recordVerifiedExecution(app, name, svcResult)
218 }
219 return result, nil
220}
221
222// Restart restarts from pinned digests without re-resolution. Empty
223// service means all services in sorted order. Every service restarts in
224// place: traffic is withdrawn, the same pinned container is restarted, its
225// readiness probe is checked, and traffic is republished. No second
226// container is created and the pinned digest never changes.
227func (s *Service) Restart(ctx context.Context, app, service, opID string) (*LifecycleResult, error) {
228 release, err := s.acquireAppContext(ctx, app)
229 if err != nil {
230 return nil, err
231 }
232 defer release()
233 return s.restartLocked(ctx, app, service, opID)
234}
235
236func (s *Service) restartLocked(ctx context.Context, app, service, opID string) (*LifecycleResult, error) {
237 ctx = lifecycleCtx(ctx, "Restart", app)
238 if err := s.deps.State.Recover(ctx); err != nil {
239 return nil, fmt.Errorf("deployment: recover before restart: %w", err)
240 }
241 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
242 return nil, err
243 }
244 active, ok, err := s.deps.State.LoadActive(ctx, app)
245 if err != nil {
246 return nil, fmt.Errorf("deployment: load active: %w", err)
247 }
248 if !ok {
249 return nil, fmt.Errorf("deployment: app %q was never deployed: %w", app, domain.ErrAppStateConflict)
250 }
251 names := sortedServiceNames(active)
252 if service != "" {
253 if _, ok := active.Services[service]; !ok {
254 return nil, fmt.Errorf("deployment: service %q not active: %w", service, domain.ErrAppStateConflict)
255 }
256 names = []string{service}
257 }
258 op := domain.AppOperation{
259 Kind: "restart", App: app,
260 StartedAt: time.Now().UTC(),
261 Request: domain.AppOperationRequestFor("restart", app, "", service),
262 }
263 op, owned, err := s.claimOperation(ctx, opID, op, nil)
264 if err != nil {
265 return nil, err
266 }
267 if !owned {
268 return replayedLifecycleResult(op), replayError(op)
269 }
270 result := &LifecycleResult{Op: op.Op, App: app, Verb: "restart", Services: map[string]ServiceResult{}}
271 var failures []string
272 for _, name := range names {
273 eff := active.Services[name]
274 // The step is journaled before the runtime work, so a candidate
275 // created by a rebuild of a missing generation is durably recorded.
276 op.Steps = append(op.Steps, domain.AppOperationStep{
277 ID: "service." + name + ".restart", State: domain.AppStepPending, Before: eff.Container, Service: name,
278 })
279 svcResult := s.restartOneService(ctx, app, op.Op, name, eff, &op, len(op.Steps)-1)
280 result.Services[name] = svcResult
281 if svcResult.Result != "deployed" {
282 failures = append(failures, name)
283 }
284 }
285 op.Outcome = ComputeOutcome(result.Services)
286 op.Warnings = journalWarnings(collectCleanupWarnings(result.Services))
287 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
288 return result, fmt.Errorf("deployment: persist restart journal: %w", err)
289 }
290 if err := s.refreshTraffic(ctx, app); err != nil {
291 s.failTrafficPublication(ctx, &op, err)
292 return result, err
293 }
294 // Routing accepted the restarted generations: their executions are
295 // verified. The app lock is held, so no periodic pass ran in between.
296 for name, svcResult := range result.Services {
297 s.recordVerifiedExecution(app, name, svcResult)
298 }
299 if len(failures) > 0 {
300 return result, fmt.Errorf("deployment: restart failed for %s: %w", strings.Join(failures, ", "), domain.ErrAppStateConflict)
301 }
302 return result, nil
303}
304
305// restartOneService withdraws one service, restarts its exact container,
306// re-inspects its binds, and verifies readiness before it may be published
307// again. Every failure leaves the service withdrawn with its recorded binds
308// cleared, so a failed restart is never served. A generation whose recorded
309// container no longer exists is rebuilt from the pinned ACTIVE digest, so a
310// failed replacement cannot leave restart as a permanent dead end.
311func (s *Service) restartOneService(ctx context.Context, app, opID, name string, eff domain.AppEffectiveService, op *domain.AppOperation, index int) ServiceResult {
312 step := &op.Steps[index]
313 result := ServiceResult{Result: "failed", Before: eff.Container, After: eff.Container}
314 fail := func(msg string) ServiceResult {
315 step.State = domain.AppStepFailed
316 step.Error = msg
317 result.Result = "failed"
318 result.Error = msg
319 return result
320 }
321 if eff.Container == "" {
322 result.After = ""
323 return fail("no active container")
324 }
325 // Fail closed before any runtime mutation if a bind's policy was
326 // revoked or became invalid since ACTIVE was published.
327 if _, err := s.resolveServiceBinds(app, eff.Spec); err != nil {
328 return fail(err.Error())
329 }
330 // A device grant revoked since ACTIVE was published must fail
331 // before any runtime mutation, under the same rule as binds. An
332 // in-place restart never rewrites device configuration.
333 if _, err := s.resolveServiceDevices(app, eff.Spec); err != nil {
334 return fail(err.Error())
335 }
336 if err := s.refuseInhibitedRestart(ctx, app, name, eff.Container); err != nil {
337 _ = s.withdrawForRecovery(ctx, app, name)
338 return fail(err.Error())
339 }
340 // Withdraw before restarting: a restarting or unverified generation
341 // must not keep receiving traffic.
342 if err := s.withdrawForRecovery(ctx, app, name); err != nil {
343 return fail(err.Error())
344 }
345 if err := s.deps.Runtime.RestartContainer(ctx, eff.Container, serviceStopGrace(eff)); err != nil {
346 if errors.Is(err, domain.ErrContainerNotFound) {
347 // The recorded generation is gone: rebuild and publish it from the
348 // pinned ACTIVE digest instead of leaving the service withdrawn
349 // until a deploy.
350 svcResult, rebuildErr := s.redeployPinned(ctx, app, opID, name, eff, s.journalCandidate(op, index))
351 if rebuildErr != nil {
352 svcResult.Error = rebuildErr.Error()
353 step.State = domain.AppStepFailed
354 step.Error = rebuildErr.Error()
355 step.Diagnostics = svcResult.Diagnostics
356 // The recorded container is proven gone: the inhibition the
357 // rebuild wrote for it protects nothing, and keeping it would
358 // refuse every later start, recovery pass, and restart.
359 if clearErr := s.clearRecoveryInhibition(ctx, app, name, eff.Container); clearErr != nil {
360 svcResult.CleanupWarnings = append(svcResult.CleanupWarnings, CleanupWarning{
361 Service: name, Leftover: eff.Container, Detail: "clear recovery inhibition: " + clearErr.Error(),
362 })
363 }
364 return svcResult
365 }
366 step.State = domain.AppStepSucceeded
367 step.After = svcResult.After
368 return svcResult
369 }
370 s.clearServiceBinds(ctx, app, name)
371 return fail(err.Error())
372 }
373 // The restarted execution is observed before readiness runs.
374 startedAt := s.executionStart(ctx, eff.Container)
375 // Ephemeral loopback binds are not guaranteed stable across a runtime
376 // restart: re-inspect before probing so the proxy never dials a stale
377 // bind.
378 binds, udpBinds, err := s.refreshBackendBinds(ctx, app, name, eff, s.deps.Traffic != nil)
379 if err != nil {
380 s.clearServiceBinds(ctx, app, name)
381 return fail(err.Error())
382 }
383 if err := s.waitServiceReady(ctx, app, eff.Container, deploymentReadiness(eff.Spec), binds); err != nil {
384 s.clearServiceBinds(ctx, app, name)
385 return fail(err.Error())
386 }
387 step.State = domain.AppStepSucceeded
388 step.After = eff.Container
389 return ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds, VerifiedStartedAt: startedAt}
390}
391
392// Remove withdraws workloads by exact container ID; volumes, secrets, and
393// ownership records are retained under the old UUID. The public name is
394// freed for reuse; a new app never implicitly adopts retained resources.
395// Stopped intent and a durable recovery inhibition per removed container
396// are persisted BEFORE any runtime effect, so native restart policy can
397// never revive a removed generation into service.
398func (s *Service) Remove(ctx context.Context, app, opID string) (*LifecycleResult, error) {
399 release, err := s.acquireAppContext(ctx, app)
400 if err != nil {
401 return nil, err
402 }
403 defer release()
404 return s.removeLocked(ctx, app, opID)
405}
406
407func (s *Service) removeLocked(ctx context.Context, app, opID string) (*LifecycleResult, error) {
408 ctx = lifecycleCtx(ctx, "Remove", app)
409 log := zerowrap.FromCtx(ctx)
410 if err := s.deps.State.Recover(ctx); err != nil {
411 return nil, fmt.Errorf("deployment: recover before remove: %w", err)
412 }
413 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
414 return nil, err
415 }
416 op := domain.AppOperation{
417 Kind: "remove", App: app,
418 StartedAt: time.Now().UTC(),
419 Request: domain.AppOperationRequestFor("remove", app, "", ""),
420 }
421 op, owned, err := s.claimOperation(ctx, opID, op, nil)
422 if err != nil {
423 return nil, err
424 }
425 if !owned {
426 return replayedLifecycleResult(op), replayError(op)
427 }
428 active, _, err := s.deps.State.LoadActive(ctx, app)
429 if err != nil {
430 activeErr := fmt.Errorf("deployment: load active: %w", err)
431 s.failOperation(ctx, &op, activeErr, "error")
432 return nil, activeErr
433 }
434 result := &LifecycleResult{Op: op.Op, App: app, Verb: "remove", Services: map[string]ServiceResult{}}
435 // Durable stopped intent and per-container inhibition precede every
436 // runtime effect: a crash between here and the container stop must
437 // never leave a generation that recovery could restart.
438 if err := s.deps.State.SaveIntent(ctx, domain.AppStopIntent{
439 App: app, Stopped: true, UpdatedBy: op.Op, UpdatedAt: time.Now().UTC(),
440 }); err != nil {
441 intentErr := fmt.Errorf("deployment: persist stopped intent before remove: %w", err)
442 s.failOperation(ctx, &op, intentErr, "error")
443 return nil, intentErr
444 }
445 steps, err := s.removeServiceContainers(ctx, app, op.Op, active, result)
446 if err != nil {
447 op.Steps = steps
448 s.failOperation(ctx, &op, err, "error")
449 return nil, err
450 }
451 // Every exact container is confirmed gone, so the app's private
452 // networks are reclaimed immediately, while the ownership record is
453 // still live and its UUID can be verified against the runtime names
454 // and labels. Shared networks stay, and retained volumes, secrets, and
455 // images are never touched.
456 ownership, err := s.deps.State.LoadOwnership(ctx, app)
457 if err != nil {
458 retireErr := fmt.Errorf("deployment: load ownership before reclamation: %w", err)
459 s.failOperation(ctx, &op, retireErr, "error")
460 return nil, retireErr
461 }
462 networkWarnings, err := s.reclaimPrivateNetworks(ctx, app, ownership)
463 if err != nil {
464 s.failOperation(ctx, &op, err, "error")
465 return nil, err
466 }
467 result.Warnings = append(result.Warnings, networkWarnings...)
468 op.Steps = steps
469 // End the incarnation atomically: the ownership record is archived
470 // with its resources retained under the old UUID, the app UUID is
471 // reset so a name reuse allocates a new incarnation, and desired,
472 // active, intent, and staged state are cleared. Without this a
473 // reapply would reuse the UUID and inherit the old secrets and
474 // volumes.
475 if err := s.deps.State.RetireApp(ctx, app); err != nil {
476 retireErr := fmt.Errorf("deployment: retire incarnation: %w", err)
477 s.failOperation(ctx, &op, retireErr, "error")
478 return nil, retireErr
479 }
480 op.Outcome = domain.AppOutcomeSuccess
481 op.Warnings = journalWarnings(append(collectCleanupWarnings(result.Services), result.Warnings...))
482 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
483 log.Warn().Err(err).Msg("deployment: failed to record remove outcome")
484 }
485 if err := s.refreshTraffic(ctx, app); err != nil {
486 s.failTrafficPublication(ctx, &op, err)
487 return result, err
488 }
489 return result, nil
490}
491
492// replayedLifecycleResult projects a replayed journal into a lifecycle
493// result without touching any workload.
494func replayedLifecycleResult(op domain.AppOperation) *LifecycleResult {
495 return &LifecycleResult{Op: op.Op, App: op.App, Verb: op.Kind, Outcome: op.Outcome}
496}
497
498// refuseInhibitedRestart blocks a restart of a generation whose recovery
499// is durably inhibited: a replacement may already have written to the
500// volume this generation still owns, so reviving it could corrupt newer
501// data. Deploying a new revision supersedes the marker safely.
502func (s *Service) refuseInhibitedRestart(ctx context.Context, app, name, containerID string) error {
503 inhibited, err := s.recoveryInhibited(ctx, app, name, containerID)
504 if err != nil {
505 return err
506 }
507 if inhibited {
508 return fmt.Errorf("deployment: restart %q/%q refused: recovery inhibited for container %s", app, name, containerID)
509 }
510 return nil
511}
512
513// removeServiceContainers persists one inhibition per removed container
514// before stopping and removing it by exact ID. Volumes are never deleted.
515// A container that could not be confirmed gone fails the remove: ACTIVE,
516// stopped intent, and the inhibition stay in place so the periodic pass
517// keeps converging the survivor instead of losing track of it.
518func (s *Service) removeServiceContainers(ctx context.Context, app, opID string, active domain.AppActive, result *LifecycleResult) ([]domain.AppOperationStep, error) {
519 var steps []domain.AppOperationStep
520 for _, name := range sortedServiceNames(active) {
521 container := active.Services[name].Container
522 step := domain.AppOperationStep{ID: "service." + name + ".remove", State: domain.AppStepPending, Before: container}
523 if container != "" {
524 if err := s.inhibitRecovery(ctx, app, name, container, "removed", opID); err != nil {
525 return nil, err
526 }
527 retired := s.retireContainer(ctx, app, retireOptions{
528 Service: name, Grace: serviceStopGrace(active.Services[name]), ClearInhibition: true,
529 }, container)
530 if !retired.Gone {
531 return nil, fmt.Errorf("deployment: remove %q/%q: container %s not confirmed gone: %s", app, name, container, cleanupDetail(retired))
532 }
533 result.Services[name] = ServiceResult{Result: domain.AppServiceRemoved, Before: container, After: "", CleanupWarnings: retired.Warnings}
534 } else {
535 result.Services[name] = ServiceResult{Result: domain.AppServiceRemoved, Before: container, After: ""}
536 }
537 step.State = domain.AppStepSucceeded
538 steps = append(steps, step)
539 }
540 if err := s.retireAllPending(ctx, app); err != nil {
541 return nil, err
542 }
543 return steps, nil
544}
545
546// retireAllPending retires every superseded container an overlap replacement
547// left running beside its successor. Ending the incarnation deletes the
548// retirement markers with the rest of the app state, so a container that is
549// not confirmed gone here would be orphaned and untracked: a failure keeps the
550// markers and fails the caller. Every marker is attempted, and a marker whose
551// claims could not be released is a failure too, so it is retried instead of
552// leaking its claims.
553func (s *Service) retireAllPending(ctx context.Context, app string) error {
554 pending, err := s.pendingRetirements(ctx, app)
555 if err != nil {
556 return err
557 }
558 var failures []error
559 for _, marker := range pending {
560 retired := s.retireContainer(ctx, app, retireOptions{
561 Service: marker.Service, Grace: marker.StopGrace, ClearInhibition: true, RetryOnLeak: true,
562 }, marker.ContainerID)
563 if !retired.Settled {
564 failures = append(failures, fmt.Errorf("deployment: remove %q/%q: superseded container %s not retired: %s", app, marker.Service, marker.ContainerID, cleanupDetail(retired)))
565 }
566 }
567 return errors.Join(failures...)
568}
569
570// pendingRetirements loads the retirement-pending markers of one app.
571func (s *Service) pendingRetirements(ctx context.Context, app string) ([]domain.AppRecoveryInhibition, error) {
572 inhibitions, err := s.deps.State.LoadRecoveryInhibitions(ctx, app)
573 if err != nil {
574 return nil, fmt.Errorf("deployment: load recovery inhibitions: %w", err)
575 }
576 _, pending := splitRecoveryInhibitions(app, inhibitions)
577 return pending, nil
578}
579
580// retirePendingForStop retires the pending superseded containers of a stopped
581// app, recording a leftover or a failure on the service they belong to. It
582// returns the services whose predecessor is not confirmed retired, and an
583// app-level error when the markers themselves could not be loaded: nothing
584// can then be said about any predecessor, which the caller must journal as a
585// failed operation.
586func (s *Service) retirePendingForStop(ctx context.Context, app string, result *LifecycleResult) ([]string, error) {
587 pending, err := s.pendingRetirements(ctx, app)
588 if err != nil {
589 result.Warnings = append(result.Warnings, CleanupWarning{Detail: "pending retirements unknown: " + err.Error()})
590 return []string{"pending retirements"}, err
591 }
592 var failed []string
593 for _, marker := range pending {
594 retired := s.retireContainer(ctx, app, retireOptions{
595 Service: marker.Service, Grace: marker.StopGrace, ClearInhibition: true, RetryOnLeak: true,
596 }, marker.ContainerID)
597 svc := result.Services[marker.Service]
598 svc.CleanupWarnings = append(svc.CleanupWarnings, retired.Warnings...)
599 if !retired.Settled {
600 svc.Result = "failed"
601 svc.Error = "superseded container " + marker.ContainerID + " not retired: " + cleanupDetail(retired)
602 failed = append(failed, marker.Service)
603 }
604 result.Services[marker.Service] = svc
605 }
606 return failed, nil
607}
608
609// ReconcileBoot reconciles apps intended to run after a daemon boot:
610// every non-stopped app with ACTIVE services is verified through Start,
611// including all-running apps: ephemeral loopback binds may shift across
612// any runtime restart while the daemon is away, so recorded binds are
613// re-inspected (never trusted) before the proxy can dial them.
614// Recovery first; no duplication of running instances; no activation of
615// pending desired revisions. Explicitly stopped apps stay stopped.
616// A failing app never blocks the verification of the following apps;
617// failures are aggregated and every unverified bind is withdrawn from
618// ACTIVE (see failServiceStep), so the post-boot host index rebuild
619// projects fail-closed for all of them.
620func (s *Service) ReconcileBoot(ctx context.Context) error {
621 ctx = lifecycleCtx(ctx, "ReconcileBoot", "boot")
622 if err := s.deps.State.Recover(ctx); err != nil {
623 return fmt.Errorf("deployment: recover before boot reconcile: %w", err)
624 }
625 apps, err := s.deps.State.ListApps(ctx)
626 if err != nil {
627 return fmt.Errorf("deployment: list apps: %w", err)
628 }
629 var failures []error
630 for _, app := range apps {
631 if err := s.reconcileBootApp(ctx, app); err != nil {
632 failures = append(failures, err)
633 }
634 }
635 return errors.Join(failures...)
636}
637
638// reconcileBootApp verifies one app at boot under the app lock. Explicitly
639// stopped apps and apps without ACTIVE services are skipped without error.
640// Stopped intent is converged: a container revived by native restart
641// policy while the daemon was away is stopped again, never restarted.
642func (s *Service) reconcileBootApp(ctx context.Context, app string) error {
643 release, err := s.acquireAppContext(ctx, app)
644 if err != nil {
645 return err
646 }
647 defer release()
648 intent, err := s.deps.State.LoadIntent(ctx, app)
649 if err != nil {
650 return fmt.Errorf("deployment: load intent: %w", err)
651 }
652 active, ok, err := s.deps.State.LoadActive(ctx, app)
653 if err != nil {
654 return fmt.Errorf("deployment: load active: %w", err)
655 }
656 if !ok || len(active.Services) == 0 {
657 // An interrupted operation is reconciled even when ACTIVE carries no
658 // service: a first deployment can be interrupted between creating its
659 // candidate and publishing it. No start follows here, so this is the
660 // only reconciliation of the boot pass.
661 return s.reconcileInterruptedDeploy(ctx, app)
662 }
663 if intent.Stopped {
664 // A stopped app does not start, so its unfinished operation is
665 // reconciled here instead.
666 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
667 return err
668 }
669 for _, name := range sortedServiceNames(active) {
670 if err := s.convergeStoppedService(ctx, app, name, active.Services[name]); err != nil {
671 return fmt.Errorf("deployment: boot stopped convergence %q/%q: %w", app, name, err)
672 }
673 }
674 return nil
675 }
676 // startLocked reconciles the app's unfinished operation before it starts
677 // anything, so the boot pass reconciles exactly once.
678 result, err := s.startLocked(ctx, app, "")
679 if err != nil {
680 return fmt.Errorf("deployment: boot start %q: %w", app, err)
681 }
682 for name, svc := range result.Services {
683 if svc.Result == "failed" {
684 return fmt.Errorf("deployment: boot verify %q/%q: %s", app, name, svc.Error)
685 }
686 }
687 return nil
688}
689
690// ensureServiceRunning starts one service when needed. Running
691// containers are bind-verified before success: ephemeral loopback binds
692// may shift across any runtime restart (stop/start, daemon absence,
693// host reboot), so the recorded binds are re-inspected and persisted
694// before the proxy can dial them. Verification failure fails the step;
695// a stale recorded bind is never served. A recorded container that no
696// longer exists is rebuilt from the revision pinned in ACTIVE.
697func (s *Service) ensureServiceRunning(ctx context.Context, app, opID, name string, eff domain.AppEffectiveService, op *domain.AppOperation, index int, result *LifecycleResult) {
698 step := &op.Steps[index]
699 if eff.Container != "" {
700 // Fail closed before any runtime mutation if a bind's policy was
701 // revoked or became invalid since ACTIVE was published.
702 if _, err := s.resolveServiceBinds(app, eff.Spec); err != nil {
703 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
704 return
705 }
706 // Revoked device grants fail before any runtime mutation.
707 if _, err := s.resolveServiceDevices(app, eff.Spec); err != nil {
708 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
709 return
710 }
711 // Boot recovery refuses a generation whose recovery is durably
712 // inhibited: a replacement may already have written to its
713 // volume, so reviving this ID could corrupt the newer data.
714 inhibited, err := s.recoveryInhibited(ctx, app, name, eff.Container)
715 if err != nil {
716 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
717 return
718 }
719 if inhibited {
720 s.failServiceStep(ctx, app, name, eff, step, result,
721 fmt.Sprintf("recovery inhibited for container %s", eff.Container))
722 return
723 }
724 if ok, err := s.deps.Runtime.IsContainerRunning(ctx, eff.Container); err == nil && ok {
725 s.verifyRunningService(ctx, app, name, eff, step, result)
726 return
727 }
728 // Restart by exact container ID when the runtime still has it.
729 if err := s.deps.Runtime.StartContainer(ctx, eff.Container); err == nil {
730 s.verifyRunningService(ctx, app, name, eff, step, result)
731 return
732 }
733 }
734 // Otherwise rebuild from the revision pinned in ACTIVE.
735 svcResult, err := s.redeployPinned(ctx, app, opID, name, eff, s.journalCandidate(op, index))
736 if err != nil {
737 step.State = domain.AppStepFailed
738 step.Error = err.Error()
739 step.Diagnostics = svcResult.Diagnostics
740 result.Services[name] = svcResult
741 return
742 }
743 step.State = domain.AppStepSucceeded
744 step.After = svcResult.After
745 result.Services[name] = svcResult
746}
747
748// redeployPinned rebuilds one service from the revision pinned in ACTIVE and
749// publishes it. It is the shared recovery path for a recorded container that
750// no longer exists: boot/start recovery, and a restart of a missing
751// generation. The returned result is always populated, so a caller can
752// journal the attempt even when err is non-nil.
753func (s *Service) redeployPinned(ctx context.Context, app, opID, name string, eff domain.AppEffectiveService, journal candidateJournal) (ServiceResult, error) {
754 rev, err := s.deps.State.LoadRevision(ctx, app, eff.EffectiveRevision)
755 if err != nil {
756 return failedServiceResult(eff.Container, err), err
757 }
758 runtimeImage, err := s.preflightImage(ctx, eff.Spec.Image, eff.Digest)
759 if err != nil {
760 return failedServiceResult(eff.Container, err), err
761 }
762 if _, err := s.resolveServiceBinds(app, eff.Spec); err != nil {
763 return failedServiceResult(eff.Container, err), err
764 }
765 // Revoked device grants fail before any runtime mutation.
766 if _, err := s.resolveServiceDevices(app, eff.Spec); err != nil {
767 return failedServiceResult(eff.Container, err), err
768 }
769 pinned := pinnedService{
770 name: name, spec: eff.Spec, digest: eff.Digest, runtimeImage: runtimeImage, appEnv: maps.Clone(rev.Spec.Env),
771 appNetworks: append([]domain.AppSharedNetwork(nil), rev.Spec.Networks...),
772 sharedNetworks: domain.AppServiceSharedNetworks(rev.Spec, name),
773 }
774 svcResult := s.deployService(ctx, app, rev.Revision, pinned, opID, eff.Container, serviceStopGrace(eff), journal)
775 if svcResult.Result == "failed" {
776 return svcResult, errors.New(svcResult.Error)
777 }
778 // The step may only be recorded as succeeded once the effective state is
779 // published: a journaled success is never ahead of what the proxy reaches.
780 if err := s.publishService(ctx, app, rev.Revision, pinned, svcResult, opID); err != nil {
781 svcResult.Result = "failed"
782 svcResult.Error = err.Error()
783 return svcResult, err
784 }
785 // The execution is deliberately not recorded here: the graph is not
786 // applied yet. Recording it before routing was accepted would let the next
787 // periodic pass see a verified, unrouted generation and skip the
788 // publication. Each caller records after its own refreshTraffic succeeded.
789 return svcResult, nil
790}
791
792// failedServiceResult builds the terminal result of a service step that
793// failed before any container existed.
794func failedServiceResult(before string, err error) ServiceResult {
795 return ServiceResult{Result: "failed", Before: before, Error: err.Error()}
796}
797
798// verifyRunningService withdraws a running or freshly started generation,
799// re-inspects its loopback binds, and verifies readiness before the service
800// may be published again. Any failure leaves the service withdrawn with its
801// recorded binds cleared, so a not-ready backend is never projected.
802func (s *Service) verifyRunningService(ctx context.Context, app, name string, eff domain.AppEffectiveService, step *domain.AppOperationStep, result *LifecycleResult) {
803 if err := s.withdrawForRecovery(ctx, app, name); err != nil {
804 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
805 return
806 }
807 startedAt := s.executionStart(ctx, eff.Container)
808 binds, udpBinds, err := s.refreshBackendBinds(ctx, app, name, eff, s.deps.Traffic != nil)
809 if err != nil {
810 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
811 return
812 }
813 if err := s.waitServiceReady(ctx, app, eff.Container, eff.Spec, binds); err != nil {
814 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
815 return
816 }
817 // The service is running and its re-inspected loopback binds are
818 // persisted, so the step may succeed: the graph apply for this app
819 // follows once, after the loop.
820 step.State = domain.AppStepSucceeded
821 step.After = eff.Container
822 result.Services[name] = ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds, VerifiedStartedAt: startedAt}
823}
824
825// refreshBackendBinds re-inspects the loopback publishes of a restarted
826// container and persists them to the current active record when needed.
827// Ephemeral binds may shift across runtime restarts; the proxy must
828// never dial the stale recorded bind. Uses the non-destructive
829// inspectBackendBinds: an existing ACTIVE container is never stopped
830// or removed by re-inspection.
831func (s *Service) refreshBackendBinds(ctx context.Context, app, name string, eff domain.AppEffectiveService, forcePersist bool) (map[int]int, map[int]int, error) {
832 ports := backendPorts(eff.Spec)
833 if len(ports) == 0 {
834 return nil, nil, nil
835 }
836 binds, udpBinds, err := s.inspectBackendBinds(ctx, app, name, eff.Container, ports)
837 if err != nil {
838 return nil, nil, err
839 }
840 if !forcePersist && equalBinds(binds, eff.BackendBinds) && equalBinds(udpBinds, eff.UDPBackendBinds) {
841 return binds, udpBinds, nil
842 }
843 active, ok, err := s.deps.State.LoadActive(ctx, app)
844 if err != nil {
845 return nil, nil, fmt.Errorf("deployment: reload active for bind refresh: %w", err)
846 }
847 if !ok {
848 return nil, nil, fmt.Errorf("deployment: app %q has no active state: %w", app, domain.ErrAppStateConflict)
849 }
850 if err := s.persistBackendBinds(ctx, active, name, eff, binds, udpBinds, forcePersist); err != nil {
851 return nil, nil, err
852 }
853 return binds, udpBinds, nil
854}
855
856func (s *Service) persistBackendBinds(ctx context.Context, active domain.AppActive, name string, eff domain.AppEffectiveService, binds, udpBinds map[int]int, force bool) error {
857 current, exists := active.Services[name]
858 if exists && current.Container != eff.Container {
859 return fmt.Errorf("deployment: active generation changed during bind refresh: %w", domain.ErrAppStateConflict)
860 }
861 if !force && exists && equalBinds(binds, current.BackendBinds) && equalBinds(udpBinds, current.UDPBackendBinds) {
862 return nil
863 }
864 if !exists {
865 current = eff
866 }
867 current.BackendBinds = binds
868 current.UDPBackendBinds = udpBinds
869 active.Services[name] = current
870 if err := s.deps.State.SaveActive(ctx, active); err != nil {
871 return fmt.Errorf("deployment: persist refreshed binds: %w", err)
872 }
873 return nil
874}
875
876// failServiceStep records a failed service step and withdraws its
877// recorded backend binds from ACTIVE: a bind that failed verification
878// (or could not be refreshed) must never be served again — the
879// projected backend would otherwise point at a dead or recycled
880// loopback port. Invalidation errors are logged, never fatal: the
881// step already failed, and the next rebuild projects fail-closed.
882func (s *Service) failServiceStep(ctx context.Context, app, name string, eff domain.AppEffectiveService, step *domain.AppOperationStep, result *LifecycleResult, errMsg string) {
883 step.State = domain.AppStepFailed
884 step.Error = errMsg
885 result.Services[name] = ServiceResult{Result: "failed", Before: eff.Container, Error: errMsg}
886 s.clearServiceBinds(ctx, app, name)
887}
888
889// clearServiceBinds withdraws a service's recorded backend binds from
890// ACTIVE through the canonical traffic boundary: a bind that failed
891// verification, refresh, or readiness must never be served, because the
892// projected backend would point at a dead, unready, or recycled loopback
893// port. No graph is applied here: the caller already owns publication
894// (or none is due). Errors are logged, never fatal: the step already
895// failed and the next rebuild projects fail-closed.
896func (s *Service) clearServiceBinds(ctx context.Context, app, name string) {
897 if s.deps.Traffic == nil {
898 return
899 }
900 if err := s.deps.Traffic.WithdrawServiceState(ctx, app, name); err != nil {
901 log := zerowrap.FromCtx(ctx)
902 log.Warn().Err(err).Str("app", app).Str("service", name).Msg("deployment: failed to withdraw unverified binds")
903 }
904}
905
906func equalBinds(a, b map[int]int) bool {
907 if len(a) != len(b) {
908 return false
909 }
910 for port, host := range a {
911 if b[port] != host {
912 return false
913 }
914 }
915 return true
916}
917func sortedServiceNames(active domain.AppActive) []string {
918 names := make([]string, 0, len(active.Services))
919 for name := range active.Services {
920 names = append(names, name)
921 }
922 sort.Strings(names)
923 return names
924}
925
926// lifecycleCtx tags lifecycle use-case logs.
927func lifecycleCtx(ctx context.Context, useCase, app string) context.Context {
928 return zerowrap.CtxWithFields(ctx, map[string]any{
929 zerowrap.FieldLayer: "usecase",
930 zerowrap.FieldUseCase: useCase,
931 "app": app,
932 })
933}