1// Package deployment implements the app deployment engine: preflight,
2// journaled execution, and terminal-result computation.
3package deployment
4
5import (
6 "context"
7 "encoding/hex"
8 "errors"
9 "fmt"
10 "maps"
11 "sort"
12 "strings"
13 "sync"
14 "time"
15
16 "github.com/bnema/zerowrap"
17 "github.com/google/uuid"
18
19 "github.com/bnema/gordon/internal/boundaries/out"
20 "github.com/bnema/gordon/internal/domain"
21 "github.com/bnema/gordon/pkg/validation"
22)
23
24// Deps are the driven adapters owned by the deployment engine.
25type Deps struct {
26 State out.AppState
27 Runtime out.ContainerRuntime
28 Images out.ImageResolver
29 Secrets out.SecretProvider
30 // Registry configures image pulls from the installation registry.
31 // Domain scopes which refs pull with auth; empty disables pulling.
32 Registry RegistryConfig
33 // Networks is the installation network policy applied to every app
34 // workload: the prefix used to derive Gordon-owned network names and
35 // whether those networks block external egress.
36 Networks NetworkConfig
37 // Limits are the installation container resource limits applied on
38 // every create and recovery path. Zero means unlimited.
39 Limits ResourceLimits
40 // ImagePolicy restricts which registries and references may be
41 // resolved, pulled, or run. It is enforced on every path, including
42 // boot, restart, and recovery.
43 ImagePolicy domain.ImageSourcePolicy
44 // Traffic is the serialized HTTP/L4 publish boundary. Rebuilt after
45 // every activation (deploy/start/restart/stop/remove) so the proxy
46 // host index follows ACTIVE; recovery also withdraws one service
47 // fail-closed before touching a workload. Nil in tests that assert
48 // state effects only. Failures are reported to the caller.
49 Traffic out.AppTrafficRefresher
50}
51
52// RegistryConfig carries the installation registry identity for pulls.
53type RegistryConfig struct {
54 Domain string
55 PullAddress string
56 Username string
57 Password string
58}
59
60// NetworkConfig carries the installation network policy for app workloads.
61type NetworkConfig struct {
62 // Prefix names the Gordon-owned networks. Empty uses the domain default.
63 Prefix string
64 // Internal blocks external egress from containers attached to those
65 // networks (Docker's Internal flag).
66 Internal bool
67}
68
69// ResourceLimits are the installation container limits applied on every
70// create and recovery path. Zero values mean unlimited.
71type ResourceLimits struct {
72 MemoryBytes int64
73 NanoCPUs int64
74 PidsLimit int64
75}
76
77// Service orchestrates captured-revision deploys.
78type Service struct {
79 deps Deps
80 log zerowrap.Logger
81 probes *ProbeDeps
82
83 // coord serializes workload mutations per app: deploy/start/
84 // restart/stop/remove/boot lock blocking, periodic reconciliation
85 // skips busy apps.
86 coord *appCoordinator
87 // barrier is the process-wide GC barrier. A shared lease covers
88 // every workload mutation from resource selection through durable
89 // protection publication; prune holds the exclusive lease.
90 barrier out.GCBarrier
91 // backoff is the in-memory, generation-scoped recovery budget.
92 backoff *recoveryBackoff
93 // publication holds the in-memory publication inhibition: a
94 // service whose fail-closed withdrawal could not be applied must be
95 // withdrawn again before any later publication.
96 publication publicationInhibition
97 // executions remembers the last observed execution start per
98 // generation, so a native restart is detected without restarting
99 // the workload ourselves.
100 executions executionTracker
101 // bindMu guards bindPolicies. A config reload replaces the whole
102 // map atomically; deploy paths read the current map without holding
103 // the lock.
104 bindMu sync.RWMutex
105 bindPolicies map[string]domain.AppBindPolicy
106 // deviceMu guards devicePolicies. A config reload replaces the whole
107 // map atomically; deploy paths read the current map without holding
108 // the lock.
109 deviceMu sync.RWMutex
110 devicePolicies map[string]domain.AppDevicePolicy
111 // liveMu guards liveOps. liveOps holds the operations this process
112 // owns between the StartDeploy claim and the end of ExecuteDeploy. It
113 // is in-memory only: a fresh Service (boot) starts empty and may
114 // reconcile every persisted non-terminal operation, while a
115 // foreground mutation must never finalize an operation a live
116 // goroutine still owns.
117 liveMu sync.Mutex
118 liveOps map[string]struct{}
119}
120
121// NewService creates the deployment engine. All deps are required;
122// Secrets may be nil only in tests that never reach secret preflight.
123func NewService(deps Deps, log zerowrap.Logger) *Service {
124 if deps.ImagePolicy.InstallationRegistry == "" {
125 deps.ImagePolicy.InstallationRegistry = deps.Registry.Domain
126 }
127 return &Service{
128 deps: deps,
129 log: log,
130 coord: newAppCoordinator(),
131 backoff: newRecoveryBackoff(time.Now),
132 }
133}
134
135// markOperationLive records that this process owns one operation from the
136// StartDeploy claim until ExecuteDeploy finishes. A live operation is the
137// single owner's in-flight work: a foreground reconciliation must not
138// finalize it, or the claim would be settled before the owner can execute it.
139func (s *Service) markOperationLive(app, op string) {
140 if app == "" || op == "" {
141 return
142 }
143 s.liveMu.Lock()
144 defer s.liveMu.Unlock()
145 if s.liveOps == nil {
146 s.liveOps = make(map[string]struct{})
147 }
148 s.liveOps[liveOperationKey(app, op)] = struct{}{}
149}
150
151// clearOperationLive drops the live marker. It is called once ExecuteDeploy
152// has recorded its outcome or left the claim recoverable, so a later
153// reconciliation can converge the operation instead of leaving it in flight.
154func (s *Service) clearOperationLive(app, op string) {
155 s.liveMu.Lock()
156 defer s.liveMu.Unlock()
157 delete(s.liveOps, liveOperationKey(app, op))
158}
159
160// operationLive reports whether this process still owns the operation.
161func (s *Service) operationLive(app, op string) bool {
162 s.liveMu.Lock()
163 defer s.liveMu.Unlock()
164 _, ok := s.liveOps[liveOperationKey(app, op)]
165 return ok
166}
167
168func liveOperationKey(app, op string) string {
169 return app + "\x00" + op
170}
171
172// WithGCBarrier wires the process-wide GC barrier. Every workload
173// mutation holds a shared lease from resource selection through the
174// durable publication of the resource's protection, so prune can never
175// observe a half-acquired resource. Nil is valid for tests that never
176// run prune.
177func (s *Service) WithGCBarrier(barrier out.GCBarrier) *Service {
178 s.barrier = barrier
179 return s
180}
181
182// WithBindPolicies supplies the initial administrative bind policies.
183func (s *Service) WithBindPolicies(policies map[string]domain.AppBindPolicy) *Service {
184 s.SetBindPolicies(policies)
185 return s
186}
187
188// SetBindPolicies atomically replaces the administrative bind policies used
189// to authorize service binds. The map is copied so later caller mutation
190// cannot race an in-flight deploy.
191func (s *Service) SetBindPolicies(policies map[string]domain.AppBindPolicy) {
192 copied := make(map[string]domain.AppBindPolicy, len(policies))
193 for name, policy := range policies {
194 copied[name] = policy
195 }
196 s.bindMu.Lock()
197 s.bindPolicies = copied
198 s.bindMu.Unlock()
199}
200
201// snapshotBindPolicies returns the current policy map. SetBindPolicies never
202// mutates a published map, so the snapshot stays safe after the lock drops.
203func (s *Service) snapshotBindPolicies() map[string]domain.AppBindPolicy {
204 s.bindMu.RLock()
205 defer s.bindMu.RUnlock()
206 return s.bindPolicies
207}
208
209// WithDevicePolicies supplies the initial administrative device policies.
210func (s *Service) WithDevicePolicies(policies map[string]domain.AppDevicePolicy) *Service {
211 s.SetDevicePolicies(policies)
212 return s
213}
214
215// SetDevicePolicies atomically replaces the administrative device policies
216// used to authorize service devices. The map and its slices are copied so
217// later caller mutation cannot race an in-flight deploy.
218func (s *Service) SetDevicePolicies(policies map[string]domain.AppDevicePolicy) {
219 copied := make(map[string]domain.AppDevicePolicy, len(policies))
220 for name, policy := range policies {
221 policy.CDI = append([]string(nil), policy.CDI...)
222 policy.AllowedApps = append([]string(nil), policy.AllowedApps...)
223 policy.AllowedServices = append([]string(nil), policy.AllowedServices...)
224 copied[name] = policy
225 }
226 s.deviceMu.Lock()
227 s.devicePolicies = copied
228 s.deviceMu.Unlock()
229}
230
231// snapshotDevicePolicies returns the current policy map. SetDevicePolicies
232// never mutates a published map, so the snapshot stays safe after the
233// lock drops.
234func (s *Service) snapshotDevicePolicies() map[string]domain.AppDevicePolicy {
235 s.deviceMu.RLock()
236 defer s.deviceMu.RUnlock()
237 return s.devicePolicies
238}
239
240// resolveServiceDevices resolves every declared logical device against the
241// current administrative policies into runtime CDI IDs. Errors name only
242// the app, service, and device: host inventory never appears.
243func (s *Service) resolveServiceDevices(app string, svc domain.AppService) ([]string, error) {
244 if len(svc.Devices) == 0 {
245 return nil, nil
246 }
247 ids, err := domain.ResolveAppDevices(app, svc.Name, svc.Devices, s.snapshotDevicePolicies())
248 if err != nil {
249 return nil, fmt.Errorf("deployment: %w", err)
250 }
251 return ids, nil
252}
253
254// resolveServiceBinds resolves every declared bind against the current
255// administrative policies into ephemeral runtime binds. Errors name only the
256// app, service, and mount: host source paths never appear.
257func (s *Service) resolveServiceBinds(app string, svc domain.AppService) ([]domain.ContainerBind, error) {
258 if len(svc.Binds) == 0 {
259 return nil, nil
260 }
261 policies := s.snapshotBindPolicies()
262 resolved := make([]domain.ContainerBind, 0, len(svc.Binds))
263 for _, bind := range svc.Binds {
264 policy, ok := policies[bind.Name]
265 if !ok {
266 return nil, fmt.Errorf("deployment: app %q service %q mount %q: no administrative mount policy configured: %w", app, svc.Name, bind.Name, domain.ErrBindPolicy)
267 }
268 r, err := policy.ResolveAppBind(app, svc.Name, bind)
269 if err != nil {
270 return nil, fmt.Errorf("deployment: app %q service %q mount %q: refused by administrative mount policy: %w", app, svc.Name, bind.Name, domain.ErrBindPolicy)
271 }
272 resolved = append(resolved, domain.ContainerBind(r))
273 }
274 return resolved, nil
275}
276
277// acquireAppContext takes the GC shared lease first and the per-app
278// coordinator second, following the documented lock order
279// (GC barrier → per-app coordinator → registry lock → bbolt
280// transaction). The returned release drops both, coordinator first.
281func (s *Service) acquireAppContext(ctx context.Context, app string) (func(), error) {
282 lease, err := s.acquireSharedGC(ctx)
283 if err != nil {
284 return nil, err
285 }
286 release, err := s.coord.acquire(ctx, app)
287 if err != nil {
288 lease.Release()
289 return nil, err
290 }
291 return func() {
292 release()
293 lease.Release()
294 }, nil
295}
296
297// tryAcquireAppContext takes the GC shared lease, then tries the per-app
298// coordinator without blocking. ok is false when the app is busy; the
299// caller skips it. Periodic reconciliation still waits for an in-flight
300// prune, because a half-planned snapshot is exactly what must not be
301// raced.
302func (s *Service) tryAcquireAppContext(ctx context.Context, app string) (func(), bool) {
303 lease, err := s.acquireSharedGC(ctx)
304 if err != nil {
305 return nil, false
306 }
307 release, ok := s.coord.tryAcquire(app)
308 if !ok {
309 lease.Release()
310 return nil, false
311 }
312 return func() {
313 release()
314 lease.Release()
315 }, true
316}
317
318// acquireSharedGC takes a shared GC lease, or a no-op lease when no
319// barrier is wired.
320func (s *Service) acquireSharedGC(ctx context.Context) (out.GCLease, error) {
321 if s.barrier == nil {
322 return noopGCLease{}, nil
323 }
324 return s.barrier.AcquireShared(ctx)
325}
326
327// noopGCLease is the lease used when no barrier is wired.
328type noopGCLease struct{}
329
330func (noopGCLease) Release() {}
331
332// WithProbeDeps overrides readiness probing (tests inject a mock-backed
333// ProbeDeps via NewProbeDeps; production leaves it nil for the live
334// runtime adapter).
335func (s *Service) WithProbeDeps(probes ProbeDeps) *Service {
336 s.probes = &probes
337 return s
338}
339
340// DeployInput selects the revision and service scope.
341type DeployInput struct {
342 App string
343 Revision string
344 Service string
345 Op string
346}
347
348// DeployResult carries terminal per-service results and cleanup warnings.
349type DeployResult struct {
350 Op string
351 App string
352 Revision string
353 Outcome string
354 Services map[string]ServiceResult
355 CleanupWarnings []CleanupWarning
356 // Removed names services retired because the new revision no longer
357 // declares them.
358 Removed []string
359}
360
361// ServiceResultUnchanged marks a deploy step that kept the running
362// container because its image, spec, and environment were already current.
363const ServiceResultUnchanged = domain.AppServiceUnchanged
364
365// ServiceResult is one service's terminal deployment result.
366type ServiceResult struct {
367 Result string
368 EffectiveRevision string
369 Before string
370 After string
371 RestartUnsafe bool
372 Error string
373 // BackendBinds carries the created container's TCP loopback publishes
374 // (container port -> 127.0.0.1 host port) for the active record.
375 BackendBinds map[int]int
376 // UDPBackendBinds carries the created container's UDP loopback
377 // publishes for the active record. Nil when the service declares
378 // no UDP interface.
379 UDPBackendBinds map[int]int
380 // CleanupWarnings are bounded leftovers of this service's terminal
381 // path: a candidate that could not be removed, or a backend claim
382 // that could not be released.
383 CleanupWarnings []CleanupWarning
384 // Diagnostics is bounded, redacted failure output of the failed
385 // candidate. It is never embedded in Error and is persisted to the
386 // journal for logs-scoped retrieval only.
387 Diagnostics []string
388}
389
390// CleanupWarning records post-publication leftovers for operator action.
391type CleanupWarning struct {
392 Service string
393 Leftover string
394 Detail string
395}
396
397// pinnedService binds a service spec to its resolved digest plus the
398// captured revision's app-wide public env (injected into every service
399// alongside that service's own resolved secrets). appEnv is a copy owned
400// by this pin: never mutated, never persisted with secret values.
401type pinnedService struct {
402 name string
403 spec domain.AppService
404 digest string
405 runtimeImage string
406 appEnv map[string]string
407 appNetworks []domain.AppSharedNetwork
408 // sharedNetworks are the declared shared-network memberships this
409 // service joins. The private incarnation network is derived from the
410 // app UUID at create time so recovery and deploy agree.
411 sharedNetworks []domain.AppSharedNetwork
412}
413
414// newOpID allocates a time-ordered op- identifier (uuid v7).
415func newOpID() string {
416 id, err := uuid.NewV7()
417 if err != nil {
418 id = uuid.New()
419 }
420 var buf [32]byte
421 hex.Encode(buf[:], id[:])
422 return "op-" + string(buf[:])
423}
424
425// claimDeploymentLocked is the first phase of a deploy: it resolves the
426// requested revision, enforces the targeted-deploy convergence rules, and
427// claims the request key, persisting the non-terminal journal. It performs
428// no image pull and no workload mutation, so a crash after it leaves a
429// durable claim that reconciliation can converge. owned is true only for the
430// single claimer of the key; a replay returns the stored journal instead.
431// The caller holds the app lock and has already recovered the store.
432func (s *Service) claimDeploymentLocked(ctx context.Context, input DeployInput) (domain.AppDesiredRevision, domain.AppOperation, bool, error) {
433 rev, err := s.resolveRevision(ctx, input)
434 if err != nil {
435 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
436 }
437 if input.Service != "" {
438 if err := s.checkConverged(ctx, input.App, input.Service, rev); err != nil {
439 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
440 }
441 }
442
443 op := domain.AppOperation{
444 Kind: "deploy",
445 App: input.App,
446 InputRevision: rev.Revision,
447 StartedAt: time.Now().UTC(),
448 Request: domain.AppOperationRequestFor("deploy", input.App, input.Revision, input.Service),
449 }
450 op, owned, err := s.claimOperation(ctx, input.Op, op, []domain.AppOperationStep{{ID: "preflight", State: domain.AppStepPending}})
451 if err != nil {
452 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
453 }
454 return rev, op, owned, nil
455}
456
457// pinPreflightLocked is the second preflight phase: it runs the image
458// pull/pin and resource gates of an already claimed operation and records the
459// pinned table (or the terminal preflight failure) in the journal. The
460// caller holds the app lock and owns the claimed, non-terminal operation.
461func (s *Service) pinPreflightLocked(ctx context.Context, app, onlyService string, rev domain.AppDesiredRevision, op *domain.AppOperation) ([]pinnedService, error) {
462 log := zerowrap.FromCtx(ctx)
463
464 pinned, err := s.preflightServices(ctx, app, rev, onlyService)
465 if err != nil {
466 op.Steps[0] = domain.AppOperationStep{ID: "preflight", State: domain.AppStepFailed, Error: err.Error()}
467 op.Outcome = domain.AppOutcomeFailed
468 if saveErr := s.deps.State.SaveOperation(ctx, *op); saveErr != nil {
469 log.Warn().Err(saveErr).Msg("deployment: failed to record preflight failure")
470 }
471 return nil, err
472 }
473 op.Steps[0] = domain.AppOperationStep{ID: "preflight", State: domain.AppStepSucceeded}
474 // A resumed claim can carry the service-step plan of an earlier attempt,
475 // including candidates it recorded. Converge those candidates before
476 // replacing the plan: an unreconciled leftover may still be running, and
477 // the replacement this execution creates must never overlap it.
478 if err := s.convergeResumedCandidates(ctx, app, op, log); err != nil {
479 return nil, err
480 }
481 // Replace the persisted plan instead of appending: the preflight step plus
482 // one step per pinned service, so runServiceStep's index contract matches a
483 // normalized layout and no earlier step is overwritten or left pending.
484 plan := make([]domain.AppOperationStep, 1, len(pinned)+1)
485 plan[0] = op.Steps[0]
486 for _, p := range pinned {
487 plan = append(plan, domain.AppOperationStep{
488 ID: "service." + p.name + ".replace",
489 State: domain.AppStepPending,
490 Service: p.name,
491 Digest: p.digest,
492 Image: p.runtimeImage,
493 })
494 }
495 op.Steps = plan
496 if err := s.deps.State.SaveOperation(ctx, *op); err != nil {
497 return nil, fmt.Errorf("deployment: persist pinned table: %w", err)
498 }
499 log.Info().Str("op", op.Op).Str("revision", rev.Revision).Int("services", len(pinned)).Msg("deployment: preflight passed")
500 return pinned, nil
501}
502
503// convergeResumedCandidates converges any candidate an earlier attempt of a
504// resumed claim recorded, so replacing its service-step plan cannot drop a
505// leftover that may still be running. A fresh claim has only its preflight
506// step and does no work here.
507func (s *Service) convergeResumedCandidates(ctx context.Context, app string, op *domain.AppOperation, log zerowrap.Logger) error {
508 needsActive := false
509 for _, step := range op.Steps {
510 if reconcileNeedsContainer(step, false) {
511 needsActive = true
512 break
513 }
514 }
515 if !needsActive {
516 return nil
517 }
518 active, _, err := s.deps.State.LoadActive(ctx, app)
519 if err != nil {
520 return fmt.Errorf("deployment: load active for resumed claim: %w", err)
521 }
522 if _, err := s.convergeInterruptedSteps(ctx, app, op, active, false, log); err != nil {
523 return err
524 }
525 return nil
526}
527
528// claimOperation is the single claim point before any mutation effect.
529// A keyed request is claimed atomically: the first claim persists the
530// in-flight journal and reports owned=true; a repeat returns the stored
531// journal and reports owned=false. A key that already answered a
532// different request fails with ErrAppStateConflict, and a key for a name
533// with no app identity fails with ErrAppNotFound without writing
534// anything. An unkeyed request (internal recovery paths) starts a fresh
535// journal under a generated id.
536func (s *Service) claimOperation(ctx context.Context, key string, op domain.AppOperation, steps []domain.AppOperationStep) (domain.AppOperation, bool, error) {
537 op.Steps = steps
538 // A journal is never opened while a different operation of this app is
539 // still unfinished: that predecessor must be reconciled first, or this
540 // claim would mask it and its leftover generation could stay untracked.
541 latest, found, err := s.deps.State.LoadLatestOperation(ctx, op.App)
542 if err != nil {
543 return domain.AppOperation{}, false, fmt.Errorf("deployment: load latest operation: %w", err)
544 }
545 if found && !latest.Terminal() && latest.Op != key {
546 return domain.AppOperation{}, false, fmt.Errorf(
547 "deployment: app %q has unfinished operation %s: %w", op.App, latest.Op, domain.ErrAppStateConflict)
548 }
549 if key == "" {
550 op.Op = newOpID()
551 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
552 return domain.AppOperation{}, false, fmt.Errorf("deployment: persist journal before effects: %w", err)
553 }
554 return op, true, nil
555 }
556 op.Op = key
557 existing, claimed, err := s.deps.State.ClaimOperation(ctx, op)
558 if err != nil {
559 return domain.AppOperation{}, false, err
560 }
561 if !claimed {
562 return existing, false, nil
563 }
564 return op, true, nil
565}
566
567// replayError reports a replayed key whose journal never reached a
568// terminal outcome: the operation is still in flight or was interrupted,
569// so its effects must not run again.
570func replayError(op domain.AppOperation) error {
571 if !op.Terminal() {
572 return fmt.Errorf("deployment: operation %s has not reached a terminal outcome: %w", op.Op, domain.ErrAppStateConflict)
573 }
574 if op.Outcome != domain.AppOutcomeSuccess {
575 return fmt.Errorf("deployment: operation %s previously ended with outcome %s: %w", op.Op, op.Outcome, domain.ErrAppStateConflict)
576 }
577 return nil
578}
579
580// resolveRevision loads the captured revision (default: current desired).
581// A revision that cannot be resolved on a name with no app identity at
582// all reports ErrAppNotFound: the failure is the missing app, not the
583// missing revision.
584func (s *Service) resolveRevision(ctx context.Context, input DeployInput) (domain.AppDesiredRevision, error) {
585 if input.Revision != "" {
586 rev, err := s.deps.State.LoadRevision(ctx, input.App, input.Revision)
587 if err != nil && errors.Is(err, domain.ErrAppRevisionNotFound) {
588 if knownErr := s.requireKnownApp(ctx, input.App); knownErr != nil {
589 return domain.AppDesiredRevision{}, knownErr
590 }
591 }
592 return rev, err
593 }
594 rev, ok, err := s.deps.State.LoadDesired(ctx, input.App)
595 if err != nil {
596 return domain.AppDesiredRevision{}, err
597 }
598 if !ok {
599 if knownErr := s.requireKnownApp(ctx, input.App); knownErr != nil {
600 return domain.AppDesiredRevision{}, knownErr
601 }
602 return domain.AppDesiredRevision{}, fmt.Errorf("deployment: app %q has no desired state: %w", input.App, domain.ErrAppRevisionNotFound)
603 }
604 return rev, nil
605}
606
607// requireKnownApp refuses a mutation of a name with no live app identity.
608func (s *Service) requireKnownApp(ctx context.Context, app string) error {
609 exists, err := s.deps.State.AppExists(ctx, app)
610 if err != nil {
611 return fmt.Errorf("deployment: app existence: %w", err)
612 }
613 if !exists {
614 return fmt.Errorf("deployment: app %q does not exist: %w", app, domain.ErrAppNotFound)
615 }
616 return nil
617}
618
619// checkConverged permits a targeted deploy when every desired/effective
620// difference belongs to that service. App-wide and other-service changes
621// must be deployed together so ACTIVE never combines incompatible specs.
622func (s *Service) checkConverged(ctx context.Context, app, service string, rev domain.AppDesiredRevision) error {
623 active, ok, err := s.deps.State.LoadActive(ctx, app)
624 if err != nil {
625 return err
626 }
627 if !ok {
628 return fmt.Errorf("deployment: app %q was never deployed, targeted deploy refused: %w", app, domain.ErrAppStateConflict)
629 }
630 if active.Converged && active.ConvergedRevision == rev.Revision {
631 return nil
632 }
633 diff := domain.DiffAppSpec(rev.Spec, effectiveAppSpec(active))
634 prefix := "service/" + service + "/"
635 if len(diff.Added) > 0 {
636 return targetedDivergenceError(rev.Revision, diff.Added[0]+" (added)")
637 }
638 if len(diff.Removed) > 0 {
639 return targetedDivergenceError(rev.Revision, diff.Removed[0]+" (removed)")
640 }
641 for _, path := range diff.Changed {
642 if !strings.HasPrefix(path, prefix) {
643 return targetedDivergenceError(rev.Revision, path)
644 }
645 }
646 return nil
647}
648
649func effectiveAppSpec(active domain.AppActive) domain.AppSpec {
650 spec := domain.AppSpec{Name: active.App, Networks: append([]domain.AppSharedNetwork(nil), active.Networks...)}
651 for name, service := range active.Services {
652 effective := service.Spec
653 effective.Name = name
654 spec.Services = append(spec.Services, effective)
655 }
656 return spec
657}
658
659func targetedDivergenceError(revision, path string) error {
660 return fmt.Errorf("%w: service-targeted deploy refused, desired %s also changes %s", domain.ErrAppStateConflict, revision, path)
661}
662
663// selectPreflightServices copies and optionally narrows a revision's services.
664func selectPreflightServices(rev domain.AppDesiredRevision, onlyService string) ([]domain.AppService, error) {
665 services := append([]domain.AppService(nil), rev.Spec.Services...)
666 if onlyService == "" {
667 return services, nil
668 }
669 for _, svc := range services {
670 if svc.Name == onlyService {
671 return []domain.AppService{svc}, nil
672 }
673 }
674 return nil, fmt.Errorf("deployment: service %q not in revision %s: %w", onlyService, rev.Revision, domain.ErrAppStateConflict)
675}
676
677// preflightServices runs the preflight gates in order. No mutation.
678func (s *Service) preflightServices(ctx context.Context, app string, rev domain.AppDesiredRevision, onlyService string) ([]pinnedService, error) {
679 services, err := selectPreflightServices(rev, onlyService)
680 if err != nil {
681 return nil, err
682 }
683 sort.Slice(services, func(i, j int) bool { return services[i].Name < services[j].Name })
684
685 var pinned []pinnedService
686 ownership, err := s.deps.State.LoadOwnership(ctx, app)
687 if err != nil {
688 return nil, err
689 }
690 // Defensive re-check: stored revisions predate or bypass apply
691 // validation; an overlap must fail preflight before any mutation.
692 if err := rev.Spec.CheckEnvSecretCollisions(); err != nil {
693 return nil, err
694 }
695 appEnv := maps.Clone(rev.Spec.Env)
696 if appEnv == nil {
697 appEnv = map[string]string{}
698 }
699 for _, svc := range services {
700 digest, err := s.deps.Images.ResolveDigest(ctx, svc.Image)
701 if err != nil {
702 return nil, fmt.Errorf("deployment: image %q unresolvable: %w", svc.Image, domain.ErrAppImageUnresolvable)
703 }
704 if err := s.checkSecrets(ctx, app, ownership.ID, svc); err != nil {
705 return nil, err
706 }
707 runtimeImage, err := s.preflightImage(ctx, svc.Image, digest)
708 if err != nil {
709 return nil, err
710 }
711 if err := s.checkImageVolumes(ctx, svc, runtimeImage); err != nil {
712 return nil, err
713 }
714 // Bind resolution stays a preflight gate: an unresolvable bind
715 // must fail before any workload mutation. The successful values
716 // are discarded because createAndStart re-resolves against the
717 // current policy immediately before the runtime mutation.
718 if _, err := s.resolveServiceBinds(app, svc); err != nil {
719 return nil, err
720 }
721 if err := s.preflightServiceDevices(ctx, app, svc, pinned); err != nil {
722 return nil, err
723 }
724 pinned = append(pinned, pinnedService{
725 name: svc.Name, spec: svc, digest: digest, runtimeImage: runtimeImage, appEnv: appEnv,
726 appNetworks: append([]domain.AppSharedNetwork(nil), rev.Spec.Networks...),
727 sharedNetworks: domain.AppServiceSharedNetworks(rev.Spec, svc.Name),
728 })
729 }
730 if err := s.recheckReservations(ctx, app, rev); err != nil {
731 return nil, err
732 }
733 if err := s.checkResources(ctx, app, ownership, services); err != nil {
734 return nil, err
735 }
736 return pinned, nil
737}
738
739// preflightServiceDevices runs the device authorization and engine
740// capability gates for one service. Split from preflightServices to keep
741// its complexity within budget.
742func (s *Service) preflightServiceDevices(ctx context.Context, app string, svc domain.AppService, pinned []pinnedService) error {
743 if _, err := s.resolveServiceDevices(app, svc); err != nil {
744 return err
745 }
746 // Engine capability is a knowable-before-mutation failure: a
747 // device-bearing revision on an unsupported engine must fail here,
748 // before withdrawal retires the serving generation. The check runs
749 // once per revision, on the first device-bearing service;
750 // CreateContainer keeps the same gate as defense.
751 if len(svc.Devices) > 0 && !devicesEngineChecked(pinned) {
752 if s.deps.Runtime == nil {
753 return fmt.Errorf("deployment: runtime unavailable: %w", domain.ErrRuntimeUnsupported)
754 }
755 // Sanitize the adapter cause: version-probe failures may embed
756 // the daemon endpoint, which must not reach journals or API
757 // responses. Cancellation and deadlines are the exception: they
758 // stay recognizable so aborted deploys are not misread as
759 // incapable engines.
760 if err := s.deps.Runtime.SupportsCDIDevices(ctx); err != nil {
761 return sanitizedEngineProbeError(svc.Name, err)
762 }
763 }
764 return nil
765}
766
767// sanitizedEngineProbeError maps an engine device-capability probe failure
768// onto the caller-visible error. Cancellation and deadlines survive so
769// shutdown and timeout handling keep working; every other cause collapses
770// to ErrRuntimeUnsupported alone, dropping the adapter text that may embed
771// the daemon endpoint.
772func sanitizedEngineProbeError(service string, err error) error {
773 switch {
774 case errors.Is(err, context.Canceled):
775 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, context.Canceled)
776 case errors.Is(err, context.DeadlineExceeded):
777 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, context.DeadlineExceeded)
778 default:
779 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, domain.ErrRuntimeUnsupported)
780 }
781}
782
783// devicesEngineChecked reports whether an earlier pinned service already
784// triggered the engine capability probe for this revision.
785func devicesEngineChecked(pinned []pinnedService) bool {
786 for _, p := range pinned {
787 if len(p.spec.Devices) > 0 {
788 return true
789 }
790 }
791 return false
792}
793
794// checkSecrets reads every required secret path once. Values stay in
795// memory for container creation only; nothing is written anywhere.
796// Paths are UUID-keyed so a removed app's secrets are never adopted
797// by a new app reusing the name.
798func (s *Service) checkSecrets(ctx context.Context, app, appID string, svc domain.AppService) error {
799 if s.deps.Secrets == nil {
800 return fmt.Errorf("deployment: secret provider unavailable: %w", domain.ErrAppSecretMissing)
801 }
802 for envKey, name := range svc.Secrets {
803 path := domain.AppSecretPathForID(appID, app, svc.Name, name)
804 if _, err := s.deps.Secrets.GetSecret(ctx, path); err != nil {
805 return fmt.Errorf("deployment: secret %s (env %s) missing: %w", path, envKey, domain.ErrAppSecretMissing)
806 }
807 }
808 return nil
809}
810
811// appSecretID returns the stable internal UUID for secret paths.
812// Empty means legacy name-keyed paths (records predating UUIDs).
813// Used on the execution path where preflight's ownership snapshot is
814// not in scope; deploys are infrequent, one extra store read per
815// service is acceptable.
816func (s *Service) appSecretID(ctx context.Context, app string) (string, error) {
817 ownership, err := s.deps.State.LoadOwnership(ctx, app)
818 if err != nil {
819 return "", err
820 }
821 return ownership.ID, nil
822}
823
824func (s *Service) preflightImage(ctx context.Context, image, digest string) (string, error) {
825 if err := validation.ValidateImageDigest(digest); err != nil {
826 return "", fmt.Errorf("deployment: image %q resolved to invalid digest: %w", image, domain.ErrAppImageUnresolvable)
827 }
828 if err := s.validateImageSource(image, digest); err != nil {
829 return "", err
830 }
831 if s.deps.Registry.Domain == "" {
832 return image, nil
833 }
834 return s.pullImage(ctx, stripImageTag(image)+"@"+digest)
835}
836
837// validateImageSource enforces the installation image policy on a
838// reference before any pull. Both the manifest reference and the pinned
839// digest are checked, so a digest-pinned ref cannot bypass the hostname
840// and port allowlist on deploy or recovery.
841func (s *Service) validateImageSource(image, digest string) error {
842 ref := strings.TrimSpace(image)
843 if digest != "" && !strings.Contains(ref, "@") {
844 ref = stripImageTag(image) + "@" + digest
845 }
846 if err := s.deps.ImagePolicy.ValidateImageSource(ref); err != nil {
847 return fmt.Errorf("deployment: image %q: %w", image, err)
848 }
849 return nil
850}
851
852// checkImageVolumes rejects images with unmapped VOLUME declarations.
853func (s *Service) checkImageVolumes(ctx context.Context, svc domain.AppService, runtimeImage string) error {
854 declared, err := s.deps.Runtime.InspectImageVolumes(ctx, runtimeImage)
855 if err != nil {
856 return fmt.Errorf("deployment: inspect image volumes: %w", err)
857 }
858 if len(declared) == 0 {
859 return nil
860 }
861 mapped := map[string]struct{}{}
862 for _, vol := range svc.Volumes {
863 mapped[vol.Path] = struct{}{}
864 }
865 // Bind destinations also map image-declared volumes: a bind that
866 // mounts over an image VOLUME is an explicit operator mapping.
867 for _, bind := range svc.Binds {
868 mapped[bind.Path] = struct{}{}
869 }
870 for _, path := range declared {
871 if _, ok := mapped[path]; !ok {
872 return fmt.Errorf(
873 "deployment: image declares unmanaged volume %q, map it explicitly: %w",
874 path, domain.ErrAppUnmanagedImageVolume,
875 )
876 }
877 }
878 return nil
879}
880
881// recheckReservations re-validates against the live global table.
882func (s *Service) recheckReservations(ctx context.Context, app string, rev domain.AppDesiredRevision) error {
883 checkpoint, err := s.deps.State.LoadCheckpoint(ctx)
884 if err != nil {
885 return err
886 }
887 reservations := domain.ReservationsFor(rev.Spec)
888 if err := domain.CheckReservations(checkpoint.Reservations, reservations, app); err != nil {
889 return fmt.Errorf("deployment: %w", err)
890 }
891 return nil
892}
893
894// checkResources verifies volumes exist-or-creatable and ownership is consistent.
895// A generated runtime volume that already exists but is NOT recorded in
896// this app's ownership is refused fail-closed: mounting it would
897// implicitly adopt (and possibly modify) foreign data. Volumes recorded
898// in ownership (attached or retained) were created by this app's own
899// deploys and are safe to reuse.
900func (s *Service) checkResources(ctx context.Context, app string, ownership domain.AppOwnership, services []domain.AppService) error {
901 owned := map[string]string{}
902 ownedRuntime := map[string]struct{}{}
903 for _, vol := range ownership.Volumes {
904 owned[vol.Name] = vol.Service
905 ownedRuntime[vol.RuntimeName] = struct{}{}
906 }
907 for _, svc := range services {
908 for _, vol := range svc.Volumes {
909 if owner, ok := owned[vol.Name]; ok && owner != svc.Name {
910 return fmt.Errorf(
911 "deployment: volume %q owned by %q, not %q: %w",
912 vol.Name, owner, svc.Name, domain.ErrAppStateConflict,
913 )
914 }
915 runtimeName := domain.RuntimeVolumeName(app, svc.Name, vol.Name)
916 exists, err := s.deps.Runtime.VolumeExists(ctx, runtimeName)
917 if err != nil {
918 return fmt.Errorf("deployment: check volume %q: %w", runtimeName, err)
919 }
920 if exists {
921 if _, ok := ownedRuntime[runtimeName]; !ok {
922 return fmt.Errorf(
923 "deployment: runtime volume %q already exists but is not owned by app %q: refusing to adopt foreign data: %w",
924 runtimeName, app, domain.ErrAppStateConflict,
925 )
926 }
927 }
928 }
929 }
930 return nil
931}
932
933// ComputeOutcome derives the op outcome over terminal per-service results.
934func ComputeOutcome(results map[string]ServiceResult) string {
935 deployed := 0
936 failed := 0
937 for _, result := range results {
938 switch result.Result {
939 case "deployed", ServiceResultUnchanged, domain.AppServiceRemoved:
940 deployed++
941 case "failed":
942 failed++
943 }
944 }
945 switch {
946 case deployed > 0 && failed > 0:
947 return domain.AppOutcomePartial
948 case deployed > 0:
949 return domain.AppOutcomeSuccess
950 default:
951 return domain.AppOutcomeFailed
952 }
953}
954
955// itoa formats integers without extra imports.
956func itoa(n int) string {
957 return fmt.Sprintf("%d", n)
958}