1package deployment
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "maps"
8 "sort"
9 "strings"
10 "time"
11
12 "github.com/bnema/zerowrap"
13
14 "github.com/bnema/gordon/internal/domain"
15)
16
17// LifecycleResult carries the terminal outcome of a lifecycle verb.
18type LifecycleResult struct {
19 Op string
20 App string
21 Verb string
22 Outcome string
23 Services map[string]ServiceResult
24 // Warnings are operation-level leftovers that belong to no single
25 // service, such as a private network that could not be verified or
26 // removed.
27 Warnings []CleanupWarning
28}
29
30// Stop persists the durable stopped intent first, then stops and removes
31// exact active containers by ID. Volumes and secrets are retained.
32func (s *Service) Stop(ctx context.Context, app, opID string) (*LifecycleResult, error) {
33 release, err := s.acquireAppContext(ctx, app)
34 if err != nil {
35 return nil, err
36 }
37 defer release()
38 return s.stopLocked(ctx, app, opID)
39}
40
41func (s *Service) stopLocked(ctx context.Context, app, opID string) (*LifecycleResult, error) {
42 ctx = lifecycleCtx(ctx, "Stop", app)
43 log := zerowrap.FromCtx(ctx)
44 if err := s.deps.State.Recover(ctx); err != nil {
45 return nil, fmt.Errorf("deployment: recover before stop: %w", err)
46 }
47 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
48 return nil, err
49 }
50 op := domain.AppOperation{
51 Kind: "stop", App: app,
52 StartedAt: time.Now().UTC(),
53 Request: domain.AppOperationRequestFor("stop", app, "", ""),
54 }
55 op, owned, err := s.claimOperation(ctx, opID, op, []domain.AppOperationStep{{ID: "intent.stopped", State: domain.AppStepPending}})
56 if err != nil {
57 return nil, err
58 }
59 if !owned {
60 return replayedLifecycleResult(op), replayError(op)
61 }
62 if err := s.deps.State.SaveIntent(ctx, domain.AppStopIntent{
63 App: app, Stopped: true, UpdatedBy: op.Op, UpdatedAt: time.Now().UTC(),
64 }); err != nil {
65 intentErr := fmt.Errorf("deployment: persist stopped intent: %w", err)
66 s.failOperation(ctx, &op, intentErr, "error")
67 return nil, intentErr
68 }
69 op.Steps[0].State = domain.AppStepSucceeded
70 active, _, err := s.deps.State.LoadActive(ctx, app)
71 if err != nil {
72 activeErr := fmt.Errorf("deployment: load active: %w", err)
73 s.failOperation(ctx, &op, activeErr, "error")
74 return nil, activeErr
75 }
76 result := &LifecycleResult{Op: op.Op, App: app, Verb: "stop", Services: map[string]ServiceResult{}}
77 var failures []string
78 for _, name := range sortedServiceNames(active) {
79 container := active.Services[name].Container
80 step, warnings, err := s.stopService(ctx, app, name, active.Services[name])
81 op.Steps = append(op.Steps, step)
82 if err != nil {
83 result.Services[name] = ServiceResult{Result: "failed", Before: container, Error: err.Error(), CleanupWarnings: warnings}
84 failures = append(failures, name)
85 continue
86 }
87 result.Services[name] = ServiceResult{Result: "deployed", Before: container, After: "", CleanupWarnings: warnings}
88 }
89 op.Outcome = ComputeOutcome(result.Services)
90 op.Warnings = journalWarnings(collectCleanupWarnings(result.Services))
91 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
92 log.Warn().Err(err).Msg("deployment: failed to record stop outcome")
93 }
94 if err := s.refreshTraffic(ctx, app); err != nil {
95 s.failTrafficPublication(ctx, &op, err)
96 return result, err
97 }
98 if len(failures) > 0 {
99 return result, fmt.Errorf("deployment: stop failed for %s: %w", strings.Join(failures, ", "), domain.ErrAppStateConflict)
100 }
101 return result, nil
102}
103
104// stopService withdraws one service, then retires its exact container
105// with the effective grace. A withdrawal or runtime failure is reported,
106// never treated as a successful stop. The confirmed disappearance
107// releases the container's backend claims and its recovery inhibition:
108// a stopped app is never revived by recovery.
109func (s *Service) stopService(ctx context.Context, app, name string, eff domain.AppEffectiveService) (domain.AppOperationStep, []CleanupWarning, error) {
110 container := eff.Container
111 step := domain.AppOperationStep{ID: "service." + name + ".stop", State: domain.AppStepPending, Before: container}
112 fail := func(err error) (domain.AppOperationStep, []CleanupWarning, error) {
113 step.State = domain.AppStepFailed
114 step.Error = err.Error()
115 return step, nil, err
116 }
117 if err := s.withdrawForRecovery(ctx, app, name); err != nil {
118 return fail(err)
119 }
120 if container != "" {
121 retired := s.retireContainer(ctx, app, retireOptions{
122 Service: name, Grace: serviceStopGrace(eff), ClearInhibition: true,
123 }, container)
124 if !retired.Gone {
125 return fail(fmt.Errorf("deployment: stop %s/%s container %s: %s", app, name, container, cleanupDetail(retired)))
126 }
127 step.After = container
128 step.State = domain.AppStepSucceeded
129 return step, retired.Warnings, nil
130 }
131 step.State = domain.AppStepSucceeded
132 return step, nil, nil
133}
134
135// Start clears the stopped intent and ensures running from active records.
136// It starts missing/stopped instances without duplicating running ones
137// and never activates pending desired revisions.
138func (s *Service) Start(ctx context.Context, app, opID string) (*LifecycleResult, error) {
139 release, err := s.acquireAppContext(ctx, app)
140 if err != nil {
141 return nil, err
142 }
143 defer release()
144 return s.startLocked(ctx, app, opID)
145}
146
147func (s *Service) startLocked(ctx context.Context, app, opID string) (*LifecycleResult, error) {
148 ctx = lifecycleCtx(ctx, "Start", app)
149 log := zerowrap.FromCtx(ctx)
150 if err := s.deps.State.Recover(ctx); err != nil {
151 return nil, fmt.Errorf("deployment: recover before start: %w", err)
152 }
153 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
154 return nil, err
155 }
156 op := domain.AppOperation{
157 Kind: "start", App: app,
158 StartedAt: time.Now().UTC(),
159 Request: domain.AppOperationRequestFor("start", app, "", ""),
160 }
161 op, owned, err := s.claimOperation(ctx, opID, op, []domain.AppOperationStep{{ID: "intent.running", State: domain.AppStepPending}})
162 if err != nil {
163 return nil, err
164 }
165 if !owned {
166 return replayedLifecycleResult(op), replayError(op)
167 }
168 if err := s.deps.State.SaveIntent(ctx, domain.AppStopIntent{
169 App: app, Stopped: false, UpdatedBy: op.Op, UpdatedAt: time.Now().UTC(),
170 }); err != nil {
171 intentErr := fmt.Errorf("deployment: clear stopped intent: %w", err)
172 s.failOperation(ctx, &op, intentErr, "error")
173 return nil, intentErr
174 }
175 op.Steps[0].State = domain.AppStepSucceeded
176 active, _, err := s.deps.State.LoadActive(ctx, app)
177 if err != nil {
178 activeErr := fmt.Errorf("deployment: load active: %w", err)
179 s.failOperation(ctx, &op, activeErr, "error")
180 return nil, activeErr
181 }
182 result := &LifecycleResult{Op: op.Op, App: app, Verb: "start", Services: map[string]ServiceResult{}}
183 for _, name := range sortedServiceNames(active) {
184 eff := active.Services[name]
185 // The step is journaled before the runtime work, so a candidate
186 // created by a redeploy is durably recorded as soon as it exists.
187 op.Steps = append(op.Steps, domain.AppOperationStep{
188 ID: "service." + name + ".start", State: domain.AppStepPending, Before: eff.Container, Service: name,
189 })
190 s.ensureServiceRunning(ctx, app, opID, name, eff, &op, len(op.Steps)-1, result)
191 }
192 op.Outcome = ComputeOutcome(result.Services)
193 op.Warnings = journalWarnings(collectCleanupWarnings(result.Services))
194 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
195 log.Warn().Err(err).Msg("deployment: failed to record start outcome")
196 }
197 if err := s.refreshTraffic(ctx, app); err != nil {
198 s.failTrafficPublication(ctx, &op, err)
199 return result, err
200 }
201 return result, nil
202}
203
204// Restart restarts from pinned digests without re-resolution. Empty
205// service means all services in sorted order. Every service restarts in
206// place: traffic is withdrawn, the same pinned container is restarted, its
207// readiness probe is checked, and traffic is republished. No second
208// container is created and the pinned digest never changes.
209func (s *Service) Restart(ctx context.Context, app, service, opID string) (*LifecycleResult, error) {
210 release, err := s.acquireAppContext(ctx, app)
211 if err != nil {
212 return nil, err
213 }
214 defer release()
215 return s.restartLocked(ctx, app, service, opID)
216}
217
218func (s *Service) restartLocked(ctx context.Context, app, service, opID string) (*LifecycleResult, error) {
219 ctx = lifecycleCtx(ctx, "Restart", app)
220 if err := s.deps.State.Recover(ctx); err != nil {
221 return nil, fmt.Errorf("deployment: recover before restart: %w", err)
222 }
223 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
224 return nil, err
225 }
226 active, ok, err := s.deps.State.LoadActive(ctx, app)
227 if err != nil {
228 return nil, fmt.Errorf("deployment: load active: %w", err)
229 }
230 if !ok {
231 return nil, fmt.Errorf("deployment: app %q was never deployed: %w", app, domain.ErrAppStateConflict)
232 }
233 names := sortedServiceNames(active)
234 if service != "" {
235 if _, ok := active.Services[service]; !ok {
236 return nil, fmt.Errorf("deployment: service %q not active: %w", service, domain.ErrAppStateConflict)
237 }
238 names = []string{service}
239 }
240 op := domain.AppOperation{
241 Kind: "restart", App: app,
242 StartedAt: time.Now().UTC(),
243 Request: domain.AppOperationRequestFor("restart", app, "", service),
244 }
245 op, owned, err := s.claimOperation(ctx, opID, op, nil)
246 if err != nil {
247 return nil, err
248 }
249 if !owned {
250 return replayedLifecycleResult(op), replayError(op)
251 }
252 result := &LifecycleResult{Op: op.Op, App: app, Verb: "restart", Services: map[string]ServiceResult{}}
253 var failures []string
254 for _, name := range names {
255 eff := active.Services[name]
256 // The step is journaled before the runtime work, so a candidate
257 // created by a rebuild of a missing generation is durably recorded.
258 op.Steps = append(op.Steps, domain.AppOperationStep{
259 ID: "service." + name + ".restart", State: domain.AppStepPending, Before: eff.Container, Service: name,
260 })
261 svcResult := s.restartOneService(ctx, app, op.Op, name, eff, &op, len(op.Steps)-1)
262 result.Services[name] = svcResult
263 if svcResult.Result != "deployed" {
264 failures = append(failures, name)
265 }
266 }
267 op.Outcome = ComputeOutcome(result.Services)
268 op.Warnings = journalWarnings(collectCleanupWarnings(result.Services))
269 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
270 return result, fmt.Errorf("deployment: persist restart journal: %w", err)
271 }
272 if err := s.refreshTraffic(ctx, app); err != nil {
273 s.failTrafficPublication(ctx, &op, err)
274 return result, err
275 }
276 if len(failures) > 0 {
277 return result, fmt.Errorf("deployment: restart failed for %s: %w", strings.Join(failures, ", "), domain.ErrAppStateConflict)
278 }
279 return result, nil
280}
281
282// restartOneService withdraws one service, restarts its exact container,
283// re-inspects its binds, and verifies readiness before it may be published
284// again. Every failure leaves the service withdrawn with its recorded binds
285// cleared, so a failed restart is never served. A generation whose recorded
286// container no longer exists is rebuilt from the pinned ACTIVE digest, so a
287// failed replacement cannot leave restart as a permanent dead end.
288func (s *Service) restartOneService(ctx context.Context, app, opID, name string, eff domain.AppEffectiveService, op *domain.AppOperation, index int) ServiceResult {
289 step := &op.Steps[index]
290 result := ServiceResult{Result: "failed", Before: eff.Container, After: eff.Container}
291 fail := func(msg string) ServiceResult {
292 step.State = domain.AppStepFailed
293 step.Error = msg
294 result.Result = "failed"
295 result.Error = msg
296 return result
297 }
298 if eff.Container == "" {
299 result.After = ""
300 return fail("no active container")
301 }
302 // Fail closed before any runtime mutation if a bind's policy was
303 // revoked or became invalid since ACTIVE was published.
304 if _, err := s.resolveServiceBinds(app, eff.Spec); err != nil {
305 return fail(err.Error())
306 }
307 // A device grant revoked since ACTIVE was published must fail
308 // before any runtime mutation, under the same rule as binds. An
309 // in-place restart never rewrites device configuration.
310 if _, err := s.resolveServiceDevices(app, eff.Spec); err != nil {
311 return fail(err.Error())
312 }
313 if err := s.refuseInhibitedRestart(ctx, app, name, eff.Container); err != nil {
314 _ = s.withdrawForRecovery(ctx, app, name)
315 return fail(err.Error())
316 }
317 // Withdraw before restarting: a restarting or unverified generation
318 // must not keep receiving traffic.
319 if err := s.withdrawForRecovery(ctx, app, name); err != nil {
320 return fail(err.Error())
321 }
322 if err := s.deps.Runtime.RestartContainer(ctx, eff.Container, serviceStopGrace(eff)); err != nil {
323 if errors.Is(err, domain.ErrContainerNotFound) {
324 // The recorded generation is gone: rebuild and publish it from the
325 // pinned ACTIVE digest instead of leaving the service withdrawn
326 // until a deploy.
327 svcResult, rebuildErr := s.redeployPinned(ctx, app, opID, name, eff, s.journalCandidate(op, index))
328 if rebuildErr != nil {
329 svcResult.Error = rebuildErr.Error()
330 step.State = domain.AppStepFailed
331 step.Error = rebuildErr.Error()
332 step.Diagnostics = svcResult.Diagnostics
333 // The recorded container is proven gone: the inhibition the
334 // rebuild wrote for it protects nothing, and keeping it would
335 // refuse every later start, recovery pass, and restart.
336 if clearErr := s.clearRecoveryInhibition(ctx, app, name, eff.Container); clearErr != nil {
337 svcResult.CleanupWarnings = append(svcResult.CleanupWarnings, CleanupWarning{
338 Service: name, Leftover: eff.Container, Detail: "clear recovery inhibition: " + clearErr.Error(),
339 })
340 }
341 return svcResult
342 }
343 step.State = domain.AppStepSucceeded
344 step.After = svcResult.After
345 return svcResult
346 }
347 s.clearServiceBinds(ctx, app, name)
348 return fail(err.Error())
349 }
350 // Ephemeral loopback binds are not guaranteed stable across a runtime
351 // restart: re-inspect before probing so the proxy never dials a stale
352 // bind.
353 binds, udpBinds, err := s.refreshBackendBinds(ctx, app, name, eff, s.deps.Traffic != nil)
354 if err != nil {
355 s.clearServiceBinds(ctx, app, name)
356 return fail(err.Error())
357 }
358 if err := s.waitServiceReady(ctx, app, eff.Container, deploymentReadiness(eff.Spec), binds); err != nil {
359 s.clearServiceBinds(ctx, app, name)
360 return fail(err.Error())
361 }
362 step.State = domain.AppStepSucceeded
363 step.After = eff.Container
364 return ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds}
365}
366
367// Remove withdraws workloads by exact container ID; volumes, secrets, and
368// ownership records are retained under the old UUID. The public name is
369// freed for reuse; a new app never implicitly adopts retained resources.
370// Stopped intent and a durable recovery inhibition per removed container
371// are persisted BEFORE any runtime effect, so native restart policy can
372// never revive a removed generation into service.
373func (s *Service) Remove(ctx context.Context, app, opID string) (*LifecycleResult, error) {
374 release, err := s.acquireAppContext(ctx, app)
375 if err != nil {
376 return nil, err
377 }
378 defer release()
379 return s.removeLocked(ctx, app, opID)
380}
381
382func (s *Service) removeLocked(ctx context.Context, app, opID string) (*LifecycleResult, error) {
383 ctx = lifecycleCtx(ctx, "Remove", app)
384 log := zerowrap.FromCtx(ctx)
385 if err := s.deps.State.Recover(ctx); err != nil {
386 return nil, fmt.Errorf("deployment: recover before remove: %w", err)
387 }
388 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
389 return nil, err
390 }
391 op := domain.AppOperation{
392 Kind: "remove", App: app,
393 StartedAt: time.Now().UTC(),
394 Request: domain.AppOperationRequestFor("remove", app, "", ""),
395 }
396 op, owned, err := s.claimOperation(ctx, opID, op, nil)
397 if err != nil {
398 return nil, err
399 }
400 if !owned {
401 return replayedLifecycleResult(op), replayError(op)
402 }
403 active, _, err := s.deps.State.LoadActive(ctx, app)
404 if err != nil {
405 activeErr := fmt.Errorf("deployment: load active: %w", err)
406 s.failOperation(ctx, &op, activeErr, "error")
407 return nil, activeErr
408 }
409 result := &LifecycleResult{Op: op.Op, App: app, Verb: "remove", Services: map[string]ServiceResult{}}
410 // Durable stopped intent and per-container inhibition precede every
411 // runtime effect: a crash between here and the container stop must
412 // never leave a generation that recovery could restart.
413 if err := s.deps.State.SaveIntent(ctx, domain.AppStopIntent{
414 App: app, Stopped: true, UpdatedBy: op.Op, UpdatedAt: time.Now().UTC(),
415 }); err != nil {
416 intentErr := fmt.Errorf("deployment: persist stopped intent before remove: %w", err)
417 s.failOperation(ctx, &op, intentErr, "error")
418 return nil, intentErr
419 }
420 steps, err := s.removeServiceContainers(ctx, app, op.Op, active, result)
421 if err != nil {
422 op.Steps = steps
423 s.failOperation(ctx, &op, err, "error")
424 return nil, err
425 }
426 // Every exact container is confirmed gone, so the app's private
427 // networks are reclaimed immediately, while the ownership record is
428 // still live and its UUID can be verified against the runtime names
429 // and labels. Shared networks stay, and retained volumes, secrets, and
430 // images are never touched.
431 ownership, err := s.deps.State.LoadOwnership(ctx, app)
432 if err != nil {
433 retireErr := fmt.Errorf("deployment: load ownership before reclamation: %w", err)
434 s.failOperation(ctx, &op, retireErr, "error")
435 return nil, retireErr
436 }
437 networkWarnings, err := s.reclaimPrivateNetworks(ctx, app, ownership)
438 if err != nil {
439 s.failOperation(ctx, &op, err, "error")
440 return nil, err
441 }
442 result.Warnings = append(result.Warnings, networkWarnings...)
443 op.Steps = steps
444 // End the incarnation atomically: the ownership record is archived
445 // with its resources retained under the old UUID, the app UUID is
446 // reset so a name reuse allocates a new incarnation, and desired,
447 // active, intent, and staged state are cleared. Without this a
448 // reapply would reuse the UUID and inherit the old secrets and
449 // volumes.
450 if err := s.deps.State.RetireApp(ctx, app); err != nil {
451 retireErr := fmt.Errorf("deployment: retire incarnation: %w", err)
452 s.failOperation(ctx, &op, retireErr, "error")
453 return nil, retireErr
454 }
455 op.Outcome = domain.AppOutcomeSuccess
456 op.Warnings = journalWarnings(append(collectCleanupWarnings(result.Services), result.Warnings...))
457 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
458 log.Warn().Err(err).Msg("deployment: failed to record remove outcome")
459 }
460 if err := s.refreshTraffic(ctx, app); err != nil {
461 s.failTrafficPublication(ctx, &op, err)
462 return result, err
463 }
464 return result, nil
465}
466
467// replayedLifecycleResult projects a replayed journal into a lifecycle
468// result without touching any workload.
469func replayedLifecycleResult(op domain.AppOperation) *LifecycleResult {
470 return &LifecycleResult{Op: op.Op, App: op.App, Verb: op.Kind, Outcome: op.Outcome}
471}
472
473// refuseInhibitedRestart blocks a restart of a generation whose recovery
474// is durably inhibited: a replacement may already have written to the
475// volume this generation still owns, so reviving it could corrupt newer
476// data. Deploying a new revision supersedes the marker safely.
477func (s *Service) refuseInhibitedRestart(ctx context.Context, app, name, containerID string) error {
478 inhibited, err := s.recoveryInhibited(ctx, app, name, containerID)
479 if err != nil {
480 return err
481 }
482 if inhibited {
483 return fmt.Errorf("deployment: restart %q/%q refused: recovery inhibited for container %s", app, name, containerID)
484 }
485 return nil
486}
487
488// removeServiceContainers persists one inhibition per removed container
489// before stopping and removing it by exact ID. Volumes are never deleted.
490// A container that could not be confirmed gone fails the remove: ACTIVE,
491// stopped intent, and the inhibition stay in place so the periodic pass
492// keeps converging the survivor instead of losing track of it.
493func (s *Service) removeServiceContainers(ctx context.Context, app, opID string, active domain.AppActive, result *LifecycleResult) ([]domain.AppOperationStep, error) {
494 var steps []domain.AppOperationStep
495 for _, name := range sortedServiceNames(active) {
496 container := active.Services[name].Container
497 step := domain.AppOperationStep{ID: "service." + name + ".remove", State: domain.AppStepPending, Before: container}
498 if container != "" {
499 if err := s.inhibitRecovery(ctx, app, name, container, "removed", opID); err != nil {
500 return nil, err
501 }
502 retired := s.retireContainer(ctx, app, retireOptions{
503 Service: name, Grace: serviceStopGrace(active.Services[name]), ClearInhibition: true,
504 }, container)
505 if !retired.Gone {
506 return nil, fmt.Errorf("deployment: remove %q/%q: container %s not confirmed gone: %s", app, name, container, cleanupDetail(retired))
507 }
508 result.Services[name] = ServiceResult{Result: "deployed", Before: container, After: "", CleanupWarnings: retired.Warnings}
509 } else {
510 result.Services[name] = ServiceResult{Result: "deployed", Before: container, After: ""}
511 }
512 step.State = domain.AppStepSucceeded
513 steps = append(steps, step)
514 }
515 return steps, nil
516}
517
518// ReconcileBoot reconciles apps intended to run after a daemon boot:
519// every non-stopped app with ACTIVE services is verified through Start,
520// including all-running apps: ephemeral loopback binds may shift across
521// any runtime restart while the daemon is away, so recorded binds are
522// re-inspected (never trusted) before the proxy can dial them.
523// Recovery first; no duplication of running instances; no activation of
524// pending desired revisions. Explicitly stopped apps stay stopped.
525// A failing app never blocks the verification of the following apps;
526// failures are aggregated and every unverified bind is withdrawn from
527// ACTIVE (see failServiceStep), so the post-boot host index rebuild
528// projects fail-closed for all of them.
529func (s *Service) ReconcileBoot(ctx context.Context) error {
530 ctx = lifecycleCtx(ctx, "ReconcileBoot", "boot")
531 if err := s.deps.State.Recover(ctx); err != nil {
532 return fmt.Errorf("deployment: recover before boot reconcile: %w", err)
533 }
534 apps, err := s.deps.State.ListApps(ctx)
535 if err != nil {
536 return fmt.Errorf("deployment: list apps: %w", err)
537 }
538 var failures []error
539 for _, app := range apps {
540 if err := s.reconcileBootApp(ctx, app); err != nil {
541 failures = append(failures, err)
542 }
543 }
544 return errors.Join(failures...)
545}
546
547// reconcileBootApp verifies one app at boot under the app lock. Explicitly
548// stopped apps and apps without ACTIVE services are skipped without error.
549// Stopped intent is converged: a container revived by native restart
550// policy while the daemon was away is stopped again, never restarted.
551func (s *Service) reconcileBootApp(ctx context.Context, app string) error {
552 release, err := s.acquireAppContext(ctx, app)
553 if err != nil {
554 return err
555 }
556 defer release()
557 intent, err := s.deps.State.LoadIntent(ctx, app)
558 if err != nil {
559 return fmt.Errorf("deployment: load intent: %w", err)
560 }
561 active, ok, err := s.deps.State.LoadActive(ctx, app)
562 if err != nil {
563 return fmt.Errorf("deployment: load active: %w", err)
564 }
565 if !ok || len(active.Services) == 0 {
566 // An interrupted operation is reconciled even when ACTIVE carries no
567 // service: a first deployment can be interrupted between creating its
568 // candidate and publishing it. No start follows here, so this is the
569 // only reconciliation of the boot pass.
570 return s.reconcileInterruptedDeploy(ctx, app)
571 }
572 if intent.Stopped {
573 // A stopped app does not start, so its unfinished operation is
574 // reconciled here instead.
575 if err := s.reconcileInterruptedDeploy(ctx, app); err != nil {
576 return err
577 }
578 for _, name := range sortedServiceNames(active) {
579 if err := s.convergeStoppedService(ctx, app, name, active.Services[name]); err != nil {
580 return fmt.Errorf("deployment: boot stopped convergence %q/%q: %w", app, name, err)
581 }
582 }
583 return nil
584 }
585 // startLocked reconciles the app's unfinished operation before it starts
586 // anything, so the boot pass reconciles exactly once.
587 result, err := s.startLocked(ctx, app, "")
588 if err != nil {
589 return fmt.Errorf("deployment: boot start %q: %w", app, err)
590 }
591 for name, svc := range result.Services {
592 if svc.Result == "failed" {
593 return fmt.Errorf("deployment: boot verify %q/%q: %s", app, name, svc.Error)
594 }
595 }
596 return nil
597}
598
599// ensureServiceRunning starts one service when needed. Running
600// containers are bind-verified before success: ephemeral loopback binds
601// may shift across any runtime restart (stop/start, daemon absence,
602// host reboot), so the recorded binds are re-inspected and persisted
603// before the proxy can dial them. Verification failure fails the step;
604// a stale recorded bind is never served. A recorded container that no
605// longer exists is rebuilt from the revision pinned in ACTIVE.
606func (s *Service) ensureServiceRunning(ctx context.Context, app, opID, name string, eff domain.AppEffectiveService, op *domain.AppOperation, index int, result *LifecycleResult) {
607 step := &op.Steps[index]
608 if eff.Container != "" {
609 // Fail closed before any runtime mutation if a bind's policy was
610 // revoked or became invalid since ACTIVE was published.
611 if _, err := s.resolveServiceBinds(app, eff.Spec); err != nil {
612 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
613 return
614 }
615 // Revoked device grants fail before any runtime mutation.
616 if _, err := s.resolveServiceDevices(app, eff.Spec); err != nil {
617 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
618 return
619 }
620 // Boot recovery refuses a generation whose recovery is durably
621 // inhibited: a replacement may already have written to its
622 // volume, so reviving this ID could corrupt the newer data.
623 inhibited, err := s.recoveryInhibited(ctx, app, name, eff.Container)
624 if err != nil {
625 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
626 return
627 }
628 if inhibited {
629 s.failServiceStep(ctx, app, name, eff, step, result,
630 fmt.Sprintf("recovery inhibited for container %s", eff.Container))
631 return
632 }
633 if ok, err := s.deps.Runtime.IsContainerRunning(ctx, eff.Container); err == nil && ok {
634 s.verifyRunningService(ctx, app, name, eff, step, result)
635 return
636 }
637 // Restart by exact container ID when the runtime still has it.
638 if err := s.deps.Runtime.StartContainer(ctx, eff.Container); err == nil {
639 s.verifyRunningService(ctx, app, name, eff, step, result)
640 return
641 }
642 }
643 // Otherwise rebuild from the revision pinned in ACTIVE.
644 svcResult, err := s.redeployPinned(ctx, app, opID, name, eff, s.journalCandidate(op, index))
645 if err != nil {
646 step.State = domain.AppStepFailed
647 step.Error = err.Error()
648 step.Diagnostics = svcResult.Diagnostics
649 result.Services[name] = svcResult
650 return
651 }
652 step.State = domain.AppStepSucceeded
653 step.After = svcResult.After
654 result.Services[name] = svcResult
655}
656
657// redeployPinned rebuilds one service from the revision pinned in ACTIVE and
658// publishes it. It is the shared recovery path for a recorded container that
659// no longer exists: boot/start recovery, and a restart of a missing
660// generation. The returned result is always populated, so a caller can
661// journal the attempt even when err is non-nil.
662func (s *Service) redeployPinned(ctx context.Context, app, opID, name string, eff domain.AppEffectiveService, journal candidateJournal) (ServiceResult, error) {
663 rev, err := s.deps.State.LoadRevision(ctx, app, eff.EffectiveRevision)
664 if err != nil {
665 return failedServiceResult(eff.Container, err), err
666 }
667 runtimeImage, err := s.preflightImage(ctx, eff.Spec.Image, eff.Digest)
668 if err != nil {
669 return failedServiceResult(eff.Container, err), err
670 }
671 if _, err := s.resolveServiceBinds(app, eff.Spec); err != nil {
672 return failedServiceResult(eff.Container, err), err
673 }
674 // Revoked device grants fail before any runtime mutation.
675 if _, err := s.resolveServiceDevices(app, eff.Spec); err != nil {
676 return failedServiceResult(eff.Container, err), err
677 }
678 pinned := pinnedService{
679 name: name, spec: eff.Spec, digest: eff.Digest, runtimeImage: runtimeImage, appEnv: maps.Clone(rev.Spec.Env),
680 appNetworks: append([]domain.AppSharedNetwork(nil), rev.Spec.Networks...),
681 sharedNetworks: domain.AppServiceSharedNetworks(rev.Spec, name),
682 }
683 svcResult := s.deployService(ctx, app, rev.Revision, pinned, opID, eff.Container, serviceStopGrace(eff), journal)
684 if svcResult.Result == "failed" {
685 return svcResult, errors.New(svcResult.Error)
686 }
687 // The step may only be recorded as succeeded once the effective state is
688 // published: a journaled success is never ahead of what the proxy reaches.
689 if err := s.publishService(ctx, app, rev.Revision, pinned, svcResult, opID); err != nil {
690 svcResult.Result = "failed"
691 svcResult.Error = err.Error()
692 return svcResult, err
693 }
694 return svcResult, nil
695}
696
697// failedServiceResult builds the terminal result of a service step that
698// failed before any container existed.
699func failedServiceResult(before string, err error) ServiceResult {
700 return ServiceResult{Result: "failed", Before: before, Error: err.Error()}
701}
702
703// verifyRunningService withdraws a running or freshly started generation,
704// re-inspects its loopback binds, and verifies readiness before the service
705// may be published again. Any failure leaves the service withdrawn with its
706// recorded binds cleared, so a not-ready backend is never projected.
707func (s *Service) verifyRunningService(ctx context.Context, app, name string, eff domain.AppEffectiveService, step *domain.AppOperationStep, result *LifecycleResult) {
708 if err := s.withdrawForRecovery(ctx, app, name); err != nil {
709 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
710 return
711 }
712 binds, udpBinds, err := s.refreshBackendBinds(ctx, app, name, eff, s.deps.Traffic != nil)
713 if err != nil {
714 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
715 return
716 }
717 if err := s.waitServiceReady(ctx, app, eff.Container, eff.Spec, binds); err != nil {
718 s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
719 return
720 }
721 // The service is running and its re-inspected loopback binds are
722 // persisted, so the step may succeed: the graph apply for this app
723 // follows once, after the loop.
724 step.State = domain.AppStepSucceeded
725 step.After = eff.Container
726 result.Services[name] = ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds}
727}
728
729// refreshBackendBinds re-inspects the loopback publishes of a restarted
730// container and persists them to the current active record when needed.
731// Ephemeral binds may shift across runtime restarts; the proxy must
732// never dial the stale recorded bind. Uses the non-destructive
733// inspectBackendBinds: an existing ACTIVE container is never stopped
734// or removed by re-inspection.
735func (s *Service) refreshBackendBinds(ctx context.Context, app, name string, eff domain.AppEffectiveService, forcePersist bool) (map[int]int, map[int]int, error) {
736 ports := backendPorts(eff.Spec)
737 if len(ports) == 0 {
738 return nil, nil, nil
739 }
740 binds, udpBinds, err := s.inspectBackendBinds(ctx, app, name, eff.Container, ports)
741 if err != nil {
742 return nil, nil, err
743 }
744 if !forcePersist && equalBinds(binds, eff.BackendBinds) && equalBinds(udpBinds, eff.UDPBackendBinds) {
745 return binds, udpBinds, nil
746 }
747 active, ok, err := s.deps.State.LoadActive(ctx, app)
748 if err != nil {
749 return nil, nil, fmt.Errorf("deployment: reload active for bind refresh: %w", err)
750 }
751 if !ok {
752 return nil, nil, fmt.Errorf("deployment: app %q has no active state: %w", app, domain.ErrAppStateConflict)
753 }
754 if err := s.persistBackendBinds(ctx, active, name, eff, binds, udpBinds, forcePersist); err != nil {
755 return nil, nil, err
756 }
757 return binds, udpBinds, nil
758}
759
760func (s *Service) persistBackendBinds(ctx context.Context, active domain.AppActive, name string, eff domain.AppEffectiveService, binds, udpBinds map[int]int, force bool) error {
761 current, exists := active.Services[name]
762 if exists && current.Container != eff.Container {
763 return fmt.Errorf("deployment: active generation changed during bind refresh: %w", domain.ErrAppStateConflict)
764 }
765 if !force && exists && equalBinds(binds, current.BackendBinds) && equalBinds(udpBinds, current.UDPBackendBinds) {
766 return nil
767 }
768 if !exists {
769 current = eff
770 }
771 current.BackendBinds = binds
772 current.UDPBackendBinds = udpBinds
773 active.Services[name] = current
774 if err := s.deps.State.SaveActive(ctx, active); err != nil {
775 return fmt.Errorf("deployment: persist refreshed binds: %w", err)
776 }
777 return nil
778}
779
780// failServiceStep records a failed service step and withdraws its
781// recorded backend binds from ACTIVE: a bind that failed verification
782// (or could not be refreshed) must never be served again — the
783// projected backend would otherwise point at a dead or recycled
784// loopback port. Invalidation errors are logged, never fatal: the
785// step already failed, and the next rebuild projects fail-closed.
786func (s *Service) failServiceStep(ctx context.Context, app, name string, eff domain.AppEffectiveService, step *domain.AppOperationStep, result *LifecycleResult, errMsg string) {
787 step.State = domain.AppStepFailed
788 step.Error = errMsg
789 result.Services[name] = ServiceResult{Result: "failed", Before: eff.Container, Error: errMsg}
790 s.clearServiceBinds(ctx, app, name)
791}
792
793// clearServiceBinds withdraws a service's recorded backend binds from
794// ACTIVE through the canonical traffic boundary: a bind that failed
795// verification, refresh, or readiness must never be served, because the
796// projected backend would point at a dead, unready, or recycled loopback
797// port. No graph is applied here: the caller already owns publication
798// (or none is due). Errors are logged, never fatal: the step already
799// failed and the next rebuild projects fail-closed.
800func (s *Service) clearServiceBinds(ctx context.Context, app, name string) {
801 if s.deps.Traffic == nil {
802 return
803 }
804 if err := s.deps.Traffic.WithdrawServiceState(ctx, app, name); err != nil {
805 log := zerowrap.FromCtx(ctx)
806 log.Warn().Err(err).Str("app", app).Str("service", name).Msg("deployment: failed to withdraw unverified binds")
807 }
808}
809
810func equalBinds(a, b map[int]int) bool {
811 if len(a) != len(b) {
812 return false
813 }
814 for port, host := range a {
815 if b[port] != host {
816 return false
817 }
818 }
819 return true
820}
821func sortedServiceNames(active domain.AppActive) []string {
822 names := make([]string, 0, len(active.Services))
823 for name := range active.Services {
824 names = append(names, name)
825 }
826 sort.Strings(names)
827 return names
828}
829
830// lifecycleCtx tags lifecycle use-case logs.
831func lifecycleCtx(ctx context.Context, useCase, app string) context.Context {
832 return zerowrap.CtxWithFields(ctx, map[string]any{
833 zerowrap.FieldLayer: "usecase",
834 zerowrap.FieldUseCase: useCase,
835 "app": app,
836 })
837}