1// Package deployment implements the app deployment engine: preflight,
2// journaled execution, and terminal-result computation.
3package deployment
4
5import (
6 "context"
7 "encoding/hex"
8 "errors"
9 "fmt"
10 "maps"
11 "sort"
12 "strings"
13 "sync"
14 "time"
15
16 "github.com/bnema/zerowrap"
17 "github.com/google/uuid"
18
19 "github.com/bnema/gordon/internal/boundaries/out"
20 "github.com/bnema/gordon/internal/domain"
21 "github.com/bnema/gordon/pkg/validation"
22)
23
24// Deps are the driven adapters owned by the deployment engine.
25type Deps struct {
26 State out.AppState
27 Runtime out.ContainerRuntime
28 Images out.ImageResolver
29 Secrets out.SecretProvider
30 // Registry configures image pulls from the installation registry.
31 // Domain scopes which refs pull with auth; empty disables pulling.
32 Registry RegistryConfig
33 // Networks is the installation network policy applied to every app
34 // workload: the prefix used to derive Gordon-owned network names and
35 // whether those networks block external egress.
36 Networks NetworkConfig
37 // Limits are the installation container resource limits applied on
38 // every create and recovery path. Zero means unlimited.
39 Limits ResourceLimits
40 // ImagePolicy restricts which registries and references may be
41 // resolved, pulled, or run. It is enforced on every path, including
42 // boot, restart, and recovery.
43 ImagePolicy domain.ImageSourcePolicy
44 // Traffic is the serialized HTTP/L4 publish boundary. Rebuilt after
45 // every activation (deploy/start/restart/stop/remove) so the proxy
46 // host index follows ACTIVE; recovery also withdraws one service
47 // fail-closed before touching a workload. Nil in tests that assert
48 // state effects only. Failures are reported to the caller.
49 Traffic out.AppTrafficRefresher
50}
51
52// RegistryConfig carries the installation registry identity for pulls.
53type RegistryConfig struct {
54 Domain string
55 PullAddress string
56 Username string
57 Password string
58}
59
60// NetworkConfig carries the installation network policy for app workloads.
61type NetworkConfig struct {
62 // Prefix names the Gordon-owned networks. Empty uses the domain default.
63 Prefix string
64 // Internal blocks external egress from containers attached to those
65 // networks (Docker's Internal flag).
66 Internal bool
67}
68
69// ResourceLimits are the installation container limits applied on every
70// create and recovery path. Zero values mean unlimited.
71type ResourceLimits struct {
72 MemoryBytes int64
73 NanoCPUs int64
74 PidsLimit int64
75}
76
77// Service orchestrates captured-revision deploys.
78type Service struct {
79 deps Deps
80 log zerowrap.Logger
81 probes *ProbeDeps
82
83 // coord serializes workload mutations per app: deploy/start/
84 // restart/stop/remove/boot lock blocking, periodic reconciliation
85 // skips busy apps.
86 coord *appCoordinator
87 // barrier is the process-wide GC barrier. A shared lease covers
88 // every workload mutation from resource selection through durable
89 // protection publication; prune holds the exclusive lease.
90 barrier out.GCBarrier
91 // backoff is the in-memory, generation-scoped recovery budget.
92 backoff *recoveryBackoff
93 // publication holds the in-memory publication inhibition: a
94 // service whose fail-closed withdrawal could not be applied must be
95 // withdrawn again before any later publication.
96 publication publicationInhibition
97 // executions remembers the last observed execution start per
98 // generation, so a native restart is detected without restarting
99 // the workload ourselves.
100 executions executionTracker
101 // bindMu guards bindPolicies. A config reload replaces the whole
102 // map atomically; deploy paths read the current map without holding
103 // the lock.
104 bindMu sync.RWMutex
105 bindPolicies map[string]domain.AppBindPolicy
106 // deviceMu guards devicePolicies. A config reload replaces the whole
107 // map atomically; deploy paths read the current map without holding
108 // the lock.
109 deviceMu sync.RWMutex
110 devicePolicies map[string]domain.AppDevicePolicy
111 // liveMu guards liveOps. liveOps holds the operations this process
112 // owns between the StartDeploy claim and the end of ExecuteDeploy. It
113 // is in-memory only: a fresh Service (boot) starts empty and may
114 // reconcile every persisted non-terminal operation, while a
115 // foreground mutation must never finalize an operation a live
116 // goroutine still owns.
117 liveMu sync.Mutex
118 liveOps map[string]struct{}
119}
120
121// NewService creates the deployment engine. All deps are required;
122// Secrets may be nil only in tests that never reach secret preflight.
123func NewService(deps Deps, log zerowrap.Logger) *Service {
124 if deps.ImagePolicy.InstallationRegistry == "" {
125 deps.ImagePolicy.InstallationRegistry = deps.Registry.Domain
126 }
127 return &Service{
128 deps: deps,
129 log: log,
130 coord: newAppCoordinator(),
131 backoff: newRecoveryBackoff(time.Now),
132 }
133}
134
135// markOperationLive records that this process owns one operation from the
136// StartDeploy claim until ExecuteDeploy finishes. A live operation is the
137// single owner's in-flight work: a foreground reconciliation must not
138// finalize it, or the claim would be settled before the owner can execute it.
139func (s *Service) markOperationLive(app, op string) {
140 if app == "" || op == "" {
141 return
142 }
143 s.liveMu.Lock()
144 defer s.liveMu.Unlock()
145 if s.liveOps == nil {
146 s.liveOps = make(map[string]struct{})
147 }
148 s.liveOps[liveOperationKey(app, op)] = struct{}{}
149}
150
151// clearOperationLive drops the live marker. It is called once ExecuteDeploy
152// has recorded its outcome or left the claim recoverable, so a later
153// reconciliation can converge the operation instead of leaving it in flight.
154func (s *Service) clearOperationLive(app, op string) {
155 s.liveMu.Lock()
156 defer s.liveMu.Unlock()
157 delete(s.liveOps, liveOperationKey(app, op))
158}
159
160// operationLive reports whether this process still owns the operation.
161func (s *Service) operationLive(app, op string) bool {
162 s.liveMu.Lock()
163 defer s.liveMu.Unlock()
164 _, ok := s.liveOps[liveOperationKey(app, op)]
165 return ok
166}
167
168func liveOperationKey(app, op string) string {
169 return app + "\x00" + op
170}
171
172// WithGCBarrier wires the process-wide GC barrier. Every workload
173// mutation holds a shared lease from resource selection through the
174// durable publication of the resource's protection, so prune can never
175// observe a half-acquired resource. Nil is valid for tests that never
176// run prune.
177func (s *Service) WithGCBarrier(barrier out.GCBarrier) *Service {
178 s.barrier = barrier
179 return s
180}
181
182// WithBindPolicies supplies the initial administrative bind policies.
183func (s *Service) WithBindPolicies(policies map[string]domain.AppBindPolicy) *Service {
184 s.SetBindPolicies(policies)
185 return s
186}
187
188// SetBindPolicies atomically replaces the administrative bind policies used
189// to authorize service binds. The map is copied so later caller mutation
190// cannot race an in-flight deploy.
191func (s *Service) SetBindPolicies(policies map[string]domain.AppBindPolicy) {
192 copied := make(map[string]domain.AppBindPolicy, len(policies))
193 for name, policy := range policies {
194 copied[name] = policy
195 }
196 s.bindMu.Lock()
197 s.bindPolicies = copied
198 s.bindMu.Unlock()
199}
200
201// snapshotBindPolicies returns the current policy map. SetBindPolicies never
202// mutates a published map, so the snapshot stays safe after the lock drops.
203func (s *Service) snapshotBindPolicies() map[string]domain.AppBindPolicy {
204 s.bindMu.RLock()
205 defer s.bindMu.RUnlock()
206 return s.bindPolicies
207}
208
209// WithDevicePolicies supplies the initial administrative device policies.
210func (s *Service) WithDevicePolicies(policies map[string]domain.AppDevicePolicy) *Service {
211 s.SetDevicePolicies(policies)
212 return s
213}
214
215// SetDevicePolicies atomically replaces the administrative device policies
216// used to authorize service devices. The map and its slices are copied so
217// later caller mutation cannot race an in-flight deploy.
218func (s *Service) SetDevicePolicies(policies map[string]domain.AppDevicePolicy) {
219 copied := make(map[string]domain.AppDevicePolicy, len(policies))
220 for name, policy := range policies {
221 policy.CDI = append([]string(nil), policy.CDI...)
222 policy.AllowedApps = append([]string(nil), policy.AllowedApps...)
223 policy.AllowedServices = append([]string(nil), policy.AllowedServices...)
224 copied[name] = policy
225 }
226 s.deviceMu.Lock()
227 s.devicePolicies = copied
228 s.deviceMu.Unlock()
229}
230
231// snapshotDevicePolicies returns the current policy map. SetDevicePolicies
232// never mutates a published map, so the snapshot stays safe after the
233// lock drops.
234func (s *Service) snapshotDevicePolicies() map[string]domain.AppDevicePolicy {
235 s.deviceMu.RLock()
236 defer s.deviceMu.RUnlock()
237 return s.devicePolicies
238}
239
240// resolveServiceDevices resolves every declared logical device against the
241// current administrative policies into runtime CDI IDs. Errors name only
242// the app, service, and device: host inventory never appears.
243func (s *Service) resolveServiceDevices(app string, svc domain.AppService) ([]string, error) {
244 if len(svc.Devices) == 0 {
245 return nil, nil
246 }
247 ids, err := domain.ResolveAppDevices(app, svc.Name, svc.Devices, s.snapshotDevicePolicies())
248 if err != nil {
249 return nil, fmt.Errorf("deployment: %w", err)
250 }
251 return ids, nil
252}
253
254// resolveServiceBinds resolves every declared bind against the current
255// administrative policies into ephemeral runtime binds. Errors name only the
256// app, service, and mount: host source paths never appear.
257func (s *Service) resolveServiceBinds(app string, svc domain.AppService) ([]domain.ContainerBind, error) {
258 if len(svc.Binds) == 0 {
259 return nil, nil
260 }
261 policies := s.snapshotBindPolicies()
262 resolved := make([]domain.ContainerBind, 0, len(svc.Binds))
263 for _, bind := range svc.Binds {
264 policy, ok := policies[bind.Name]
265 if !ok {
266 return nil, fmt.Errorf("deployment: app %q service %q mount %q: no administrative mount policy configured: %w", app, svc.Name, bind.Name, domain.ErrBindPolicy)
267 }
268 r, err := policy.ResolveAppBind(app, svc.Name, bind)
269 if err != nil {
270 return nil, fmt.Errorf("deployment: app %q service %q mount %q: refused by administrative mount policy: %w", app, svc.Name, bind.Name, domain.ErrBindPolicy)
271 }
272 resolved = append(resolved, domain.ContainerBind(r))
273 }
274 return resolved, nil
275}
276
277// acquireAppContext takes the GC shared lease first and the per-app
278// coordinator second, following the documented lock order
279// (GC barrier → per-app coordinator → registry lock → bbolt
280// transaction). The returned release drops both, coordinator first.
281func (s *Service) acquireAppContext(ctx context.Context, app string) (func(), error) {
282 lease, err := s.acquireSharedGC(ctx)
283 if err != nil {
284 return nil, err
285 }
286 release, err := s.coord.acquire(ctx, app)
287 if err != nil {
288 lease.Release()
289 return nil, err
290 }
291 return func() {
292 release()
293 lease.Release()
294 }, nil
295}
296
297// tryAcquireAppContext takes the GC shared lease, then tries the per-app
298// coordinator without blocking. ok is false when the app is busy; the
299// caller skips it. Periodic reconciliation still waits for an in-flight
300// prune, because a half-planned snapshot is exactly what must not be
301// raced.
302func (s *Service) tryAcquireAppContext(ctx context.Context, app string) (func(), bool) {
303 lease, err := s.acquireSharedGC(ctx)
304 if err != nil {
305 return nil, false
306 }
307 release, ok := s.coord.tryAcquire(app)
308 if !ok {
309 lease.Release()
310 return nil, false
311 }
312 return func() {
313 release()
314 lease.Release()
315 }, true
316}
317
318// acquireSharedGC takes a shared GC lease, or a no-op lease when no
319// barrier is wired.
320func (s *Service) acquireSharedGC(ctx context.Context) (out.GCLease, error) {
321 if s.barrier == nil {
322 return noopGCLease{}, nil
323 }
324 return s.barrier.AcquireShared(ctx)
325}
326
327// noopGCLease is the lease used when no barrier is wired.
328type noopGCLease struct{}
329
330func (noopGCLease) Release() {}
331
332// WithProbeDeps overrides readiness probing (tests inject a mock-backed
333// ProbeDeps via NewProbeDeps; production leaves it nil for the live
334// runtime adapter).
335func (s *Service) WithProbeDeps(probes ProbeDeps) *Service {
336 s.probes = &probes
337 return s
338}
339
340// DeployInput selects the revision and service scope.
341type DeployInput struct {
342 App string
343 Revision string
344 Service string
345 Op string
346}
347
348// DeployResult carries terminal per-service results and cleanup warnings.
349type DeployResult struct {
350 Op string
351 App string
352 Revision string
353 Outcome string
354 Services map[string]ServiceResult
355 CleanupWarnings []CleanupWarning
356 // Removed names services retired because the new revision no longer
357 // declares them.
358 Removed []string
359}
360
361// ServiceResult is one service's terminal deployment result.
362type ServiceResult struct {
363 Result string
364 EffectiveRevision string
365 Before string
366 After string
367 RestartUnsafe bool
368 Error string
369 // BackendBinds carries the created container's TCP loopback publishes
370 // (container port -> 127.0.0.1 host port) for the active record.
371 BackendBinds map[int]int
372 // UDPBackendBinds carries the created container's UDP loopback
373 // publishes for the active record. Nil when the service declares
374 // no UDP interface.
375 UDPBackendBinds map[int]int
376 // CleanupWarnings are bounded leftovers of this service's terminal
377 // path: a candidate that could not be removed, or a backend claim
378 // that could not be released.
379 CleanupWarnings []CleanupWarning
380 // Diagnostics is bounded, redacted failure output of the failed
381 // candidate. It is never embedded in Error and is persisted to the
382 // journal for logs-scoped retrieval only.
383 Diagnostics []string
384}
385
386// CleanupWarning records post-publication leftovers for operator action.
387type CleanupWarning struct {
388 Service string
389 Leftover string
390 Detail string
391}
392
393// pinnedService binds a service spec to its resolved digest plus the
394// captured revision's app-wide public env (injected into every service
395// alongside that service's own resolved secrets). appEnv is a copy owned
396// by this pin: never mutated, never persisted with secret values.
397type pinnedService struct {
398 name string
399 spec domain.AppService
400 digest string
401 runtimeImage string
402 appEnv map[string]string
403 appNetworks []domain.AppSharedNetwork
404 // sharedNetworks are the declared shared-network memberships this
405 // service joins. The private incarnation network is derived from the
406 // app UUID at create time so recovery and deploy agree.
407 sharedNetworks []domain.AppSharedNetwork
408}
409
410// newOpID allocates a time-ordered op- identifier (uuid v7).
411func newOpID() string {
412 id, err := uuid.NewV7()
413 if err != nil {
414 id = uuid.New()
415 }
416 var buf [32]byte
417 hex.Encode(buf[:], id[:])
418 return "op-" + string(buf[:])
419}
420
421// claimDeploymentLocked is the first phase of a deploy: it resolves the
422// requested revision, enforces the targeted-deploy convergence rules, and
423// claims the request key, persisting the non-terminal journal. It performs
424// no image pull and no workload mutation, so a crash after it leaves a
425// durable claim that reconciliation can converge. owned is true only for the
426// single claimer of the key; a replay returns the stored journal instead.
427// The caller holds the app lock and has already recovered the store.
428func (s *Service) claimDeploymentLocked(ctx context.Context, input DeployInput) (domain.AppDesiredRevision, domain.AppOperation, bool, error) {
429 rev, err := s.resolveRevision(ctx, input)
430 if err != nil {
431 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
432 }
433 if input.Service != "" {
434 if err := s.checkConverged(ctx, input.App, input.Service, rev); err != nil {
435 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
436 }
437 }
438
439 op := domain.AppOperation{
440 Kind: "deploy",
441 App: input.App,
442 InputRevision: rev.Revision,
443 StartedAt: time.Now().UTC(),
444 Request: domain.AppOperationRequestFor("deploy", input.App, input.Revision, input.Service),
445 }
446 op, owned, err := s.claimOperation(ctx, input.Op, op, []domain.AppOperationStep{{ID: "preflight", State: domain.AppStepPending}})
447 if err != nil {
448 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
449 }
450 return rev, op, owned, nil
451}
452
453// pinPreflightLocked is the second preflight phase: it runs the image
454// pull/pin and resource gates of an already claimed operation and records the
455// pinned table (or the terminal preflight failure) in the journal. The
456// caller holds the app lock and owns the claimed, non-terminal operation.
457func (s *Service) pinPreflightLocked(ctx context.Context, app, onlyService string, rev domain.AppDesiredRevision, op *domain.AppOperation) ([]pinnedService, error) {
458 log := zerowrap.FromCtx(ctx)
459
460 pinned, err := s.preflightServices(ctx, app, rev, onlyService)
461 if err != nil {
462 op.Steps[0] = domain.AppOperationStep{ID: "preflight", State: domain.AppStepFailed, Error: err.Error()}
463 op.Outcome = domain.AppOutcomeFailed
464 if saveErr := s.deps.State.SaveOperation(ctx, *op); saveErr != nil {
465 log.Warn().Err(saveErr).Msg("deployment: failed to record preflight failure")
466 }
467 return nil, err
468 }
469 op.Steps[0] = domain.AppOperationStep{ID: "preflight", State: domain.AppStepSucceeded}
470 // A resumed claim can carry the service-step plan of an earlier attempt,
471 // including candidates it recorded. Converge those candidates before
472 // replacing the plan: an unreconciled leftover may still be running, and
473 // the replacement this execution creates must never overlap it.
474 if err := s.convergeResumedCandidates(ctx, app, op, log); err != nil {
475 return nil, err
476 }
477 // Replace the persisted plan instead of appending: the preflight step plus
478 // one step per pinned service, so runServiceStep's index contract matches a
479 // normalized layout and no earlier step is overwritten or left pending.
480 plan := make([]domain.AppOperationStep, 1, len(pinned)+1)
481 plan[0] = op.Steps[0]
482 for _, p := range pinned {
483 plan = append(plan, domain.AppOperationStep{
484 ID: "service." + p.name + ".replace",
485 State: domain.AppStepPending,
486 Service: p.name,
487 Digest: p.digest,
488 Image: p.runtimeImage,
489 })
490 }
491 op.Steps = plan
492 if err := s.deps.State.SaveOperation(ctx, *op); err != nil {
493 return nil, fmt.Errorf("deployment: persist pinned table: %w", err)
494 }
495 log.Info().Str("op", op.Op).Str("revision", rev.Revision).Int("services", len(pinned)).Msg("deployment: preflight passed")
496 return pinned, nil
497}
498
499// convergeResumedCandidates converges any candidate an earlier attempt of a
500// resumed claim recorded, so replacing its service-step plan cannot drop a
501// leftover that may still be running. A fresh claim has only its preflight
502// step and does no work here.
503func (s *Service) convergeResumedCandidates(ctx context.Context, app string, op *domain.AppOperation, log zerowrap.Logger) error {
504 needsActive := false
505 for _, step := range op.Steps {
506 if reconcileNeedsContainer(step, false) {
507 needsActive = true
508 break
509 }
510 }
511 if !needsActive {
512 return nil
513 }
514 active, _, err := s.deps.State.LoadActive(ctx, app)
515 if err != nil {
516 return fmt.Errorf("deployment: load active for resumed claim: %w", err)
517 }
518 if _, err := s.convergeInterruptedSteps(ctx, app, op, active, false, log); err != nil {
519 return err
520 }
521 return nil
522}
523
524// claimOperation is the single claim point before any mutation effect.
525// A keyed request is claimed atomically: the first claim persists the
526// in-flight journal and reports owned=true; a repeat returns the stored
527// journal and reports owned=false. A key that already answered a
528// different request fails with ErrAppStateConflict, and a key for a name
529// with no app identity fails with ErrAppNotFound without writing
530// anything. An unkeyed request (internal recovery paths) starts a fresh
531// journal under a generated id.
532func (s *Service) claimOperation(ctx context.Context, key string, op domain.AppOperation, steps []domain.AppOperationStep) (domain.AppOperation, bool, error) {
533 op.Steps = steps
534 // A journal is never opened while a different operation of this app is
535 // still unfinished: that predecessor must be reconciled first, or this
536 // claim would mask it and its leftover generation could stay untracked.
537 latest, found, err := s.deps.State.LoadLatestOperation(ctx, op.App)
538 if err != nil {
539 return domain.AppOperation{}, false, fmt.Errorf("deployment: load latest operation: %w", err)
540 }
541 if found && !latest.Terminal() && latest.Op != key {
542 return domain.AppOperation{}, false, fmt.Errorf(
543 "deployment: app %q has unfinished operation %s: %w", op.App, latest.Op, domain.ErrAppStateConflict)
544 }
545 if key == "" {
546 op.Op = newOpID()
547 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
548 return domain.AppOperation{}, false, fmt.Errorf("deployment: persist journal before effects: %w", err)
549 }
550 return op, true, nil
551 }
552 op.Op = key
553 existing, claimed, err := s.deps.State.ClaimOperation(ctx, op)
554 if err != nil {
555 return domain.AppOperation{}, false, err
556 }
557 if !claimed {
558 return existing, false, nil
559 }
560 return op, true, nil
561}
562
563// replayError reports a replayed key whose journal never reached a
564// terminal outcome: the operation is still in flight or was interrupted,
565// so its effects must not run again.
566func replayError(op domain.AppOperation) error {
567 if !op.Terminal() {
568 return fmt.Errorf("deployment: operation %s has not reached a terminal outcome: %w", op.Op, domain.ErrAppStateConflict)
569 }
570 if op.Outcome != domain.AppOutcomeSuccess {
571 return fmt.Errorf("deployment: operation %s previously ended with outcome %s: %w", op.Op, op.Outcome, domain.ErrAppStateConflict)
572 }
573 return nil
574}
575
576// resolveRevision loads the captured revision (default: current desired).
577// A revision that cannot be resolved on a name with no app identity at
578// all reports ErrAppNotFound: the failure is the missing app, not the
579// missing revision.
580func (s *Service) resolveRevision(ctx context.Context, input DeployInput) (domain.AppDesiredRevision, error) {
581 if input.Revision != "" {
582 rev, err := s.deps.State.LoadRevision(ctx, input.App, input.Revision)
583 if err != nil && errors.Is(err, domain.ErrAppRevisionNotFound) {
584 if knownErr := s.requireKnownApp(ctx, input.App); knownErr != nil {
585 return domain.AppDesiredRevision{}, knownErr
586 }
587 }
588 return rev, err
589 }
590 rev, ok, err := s.deps.State.LoadDesired(ctx, input.App)
591 if err != nil {
592 return domain.AppDesiredRevision{}, err
593 }
594 if !ok {
595 if knownErr := s.requireKnownApp(ctx, input.App); knownErr != nil {
596 return domain.AppDesiredRevision{}, knownErr
597 }
598 return domain.AppDesiredRevision{}, fmt.Errorf("deployment: app %q has no desired state: %w", input.App, domain.ErrAppRevisionNotFound)
599 }
600 return rev, nil
601}
602
603// requireKnownApp refuses a mutation of a name with no live app identity.
604func (s *Service) requireKnownApp(ctx context.Context, app string) error {
605 exists, err := s.deps.State.AppExists(ctx, app)
606 if err != nil {
607 return fmt.Errorf("deployment: app existence: %w", err)
608 }
609 if !exists {
610 return fmt.Errorf("deployment: app %q does not exist: %w", app, domain.ErrAppNotFound)
611 }
612 return nil
613}
614
615// checkConverged permits a targeted deploy when every desired/effective
616// difference belongs to that service. App-wide and other-service changes
617// must be deployed together so ACTIVE never combines incompatible specs.
618func (s *Service) checkConverged(ctx context.Context, app, service string, rev domain.AppDesiredRevision) error {
619 active, ok, err := s.deps.State.LoadActive(ctx, app)
620 if err != nil {
621 return err
622 }
623 if !ok {
624 return fmt.Errorf("deployment: app %q was never deployed, targeted deploy refused: %w", app, domain.ErrAppStateConflict)
625 }
626 if active.Converged && active.ConvergedRevision == rev.Revision {
627 return nil
628 }
629 diff := domain.DiffAppSpec(rev.Spec, effectiveAppSpec(active))
630 prefix := "service/" + service + "/"
631 if len(diff.Added) == 0 && len(diff.Removed) == 0 && len(diff.Changed) > 0 {
632 for _, path := range diff.Changed {
633 if !strings.HasPrefix(path, prefix) {
634 return targetedDivergenceError(rev.Revision, path)
635 }
636 }
637 return nil
638 }
639 return targetedDivergenceError(rev.Revision, "application structure")
640}
641
642func effectiveAppSpec(active domain.AppActive) domain.AppSpec {
643 spec := domain.AppSpec{Name: active.App, Networks: append([]domain.AppSharedNetwork(nil), active.Networks...)}
644 for name, service := range active.Services {
645 effective := service.Spec
646 effective.Name = name
647 spec.Services = append(spec.Services, effective)
648 }
649 return spec
650}
651
652func targetedDivergenceError(revision, path string) error {
653 return fmt.Errorf("%w: service-targeted deploy refused, desired %s also changes %s", domain.ErrAppStateConflict, revision, path)
654}
655
656// selectPreflightServices copies and optionally narrows a revision's services.
657func selectPreflightServices(rev domain.AppDesiredRevision, onlyService string) ([]domain.AppService, error) {
658 services := append([]domain.AppService(nil), rev.Spec.Services...)
659 if onlyService == "" {
660 return services, nil
661 }
662 for _, svc := range services {
663 if svc.Name == onlyService {
664 return []domain.AppService{svc}, nil
665 }
666 }
667 return nil, fmt.Errorf("deployment: service %q not in revision %s: %w", onlyService, rev.Revision, domain.ErrAppStateConflict)
668}
669
670// preflightServices runs the preflight gates in order. No mutation.
671func (s *Service) preflightServices(ctx context.Context, app string, rev domain.AppDesiredRevision, onlyService string) ([]pinnedService, error) {
672 services, err := selectPreflightServices(rev, onlyService)
673 if err != nil {
674 return nil, err
675 }
676 sort.Slice(services, func(i, j int) bool { return services[i].Name < services[j].Name })
677
678 var pinned []pinnedService
679 ownership, err := s.deps.State.LoadOwnership(ctx, app)
680 if err != nil {
681 return nil, err
682 }
683 // Defensive re-check: stored revisions predate or bypass apply
684 // validation; an overlap must fail preflight before any mutation.
685 if err := rev.Spec.CheckEnvSecretCollisions(); err != nil {
686 return nil, err
687 }
688 appEnv := maps.Clone(rev.Spec.Env)
689 if appEnv == nil {
690 appEnv = map[string]string{}
691 }
692 for _, svc := range services {
693 digest, err := s.deps.Images.ResolveDigest(ctx, svc.Image)
694 if err != nil {
695 return nil, fmt.Errorf("deployment: image %q unresolvable: %w", svc.Image, domain.ErrAppImageUnresolvable)
696 }
697 if err := s.checkSecrets(ctx, app, ownership.ID, svc); err != nil {
698 return nil, err
699 }
700 runtimeImage, err := s.preflightImage(ctx, svc.Image, digest)
701 if err != nil {
702 return nil, err
703 }
704 if err := s.checkImageVolumes(ctx, svc, runtimeImage); err != nil {
705 return nil, err
706 }
707 // Bind resolution stays a preflight gate: an unresolvable bind
708 // must fail before any workload mutation. The successful values
709 // are discarded because createAndStart re-resolves against the
710 // current policy immediately before the runtime mutation.
711 if _, err := s.resolveServiceBinds(app, svc); err != nil {
712 return nil, err
713 }
714 if err := s.preflightServiceDevices(ctx, app, svc, pinned); err != nil {
715 return nil, err
716 }
717 pinned = append(pinned, pinnedService{
718 name: svc.Name, spec: svc, digest: digest, runtimeImage: runtimeImage, appEnv: appEnv,
719 appNetworks: append([]domain.AppSharedNetwork(nil), rev.Spec.Networks...),
720 sharedNetworks: domain.AppServiceSharedNetworks(rev.Spec, svc.Name),
721 })
722 }
723 if err := s.recheckReservations(ctx, app, rev); err != nil {
724 return nil, err
725 }
726 if err := s.checkResources(ctx, app, ownership, services); err != nil {
727 return nil, err
728 }
729 return pinned, nil
730}
731
732// preflightServiceDevices runs the device authorization and engine
733// capability gates for one service. Split from preflightServices to keep
734// its complexity within budget.
735func (s *Service) preflightServiceDevices(ctx context.Context, app string, svc domain.AppService, pinned []pinnedService) error {
736 if _, err := s.resolveServiceDevices(app, svc); err != nil {
737 return err
738 }
739 // Engine capability is a knowable-before-mutation failure: a
740 // device-bearing revision on an unsupported engine must fail here,
741 // before withdrawal retires the serving generation. The check runs
742 // once per revision, on the first device-bearing service;
743 // CreateContainer keeps the same gate as defense.
744 if len(svc.Devices) > 0 && !devicesEngineChecked(pinned) {
745 if s.deps.Runtime == nil {
746 return fmt.Errorf("deployment: runtime unavailable: %w", domain.ErrRuntimeUnsupported)
747 }
748 // Sanitize the adapter cause: version-probe failures may embed
749 // the daemon endpoint, which must not reach journals or API
750 // responses. Cancellation and deadlines are the exception: they
751 // stay recognizable so aborted deploys are not misread as
752 // incapable engines.
753 if err := s.deps.Runtime.SupportsCDIDevices(ctx); err != nil {
754 return sanitizedEngineProbeError(svc.Name, err)
755 }
756 }
757 return nil
758}
759
760// sanitizedEngineProbeError maps an engine device-capability probe failure
761// onto the caller-visible error. Cancellation and deadlines survive so
762// shutdown and timeout handling keep working; every other cause collapses
763// to ErrRuntimeUnsupported alone, dropping the adapter text that may embed
764// the daemon endpoint.
765func sanitizedEngineProbeError(service string, err error) error {
766 switch {
767 case errors.Is(err, context.Canceled):
768 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, context.Canceled)
769 case errors.Is(err, context.DeadlineExceeded):
770 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, context.DeadlineExceeded)
771 default:
772 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, domain.ErrRuntimeUnsupported)
773 }
774}
775
776// devicesEngineChecked reports whether an earlier pinned service already
777// triggered the engine capability probe for this revision.
778func devicesEngineChecked(pinned []pinnedService) bool {
779 for _, p := range pinned {
780 if len(p.spec.Devices) > 0 {
781 return true
782 }
783 }
784 return false
785}
786
787// checkSecrets reads every required secret path once. Values stay in
788// memory for container creation only; nothing is written anywhere.
789// Paths are UUID-keyed so a removed app's secrets are never adopted
790// by a new app reusing the name.
791func (s *Service) checkSecrets(ctx context.Context, app, appID string, svc domain.AppService) error {
792 if s.deps.Secrets == nil {
793 return fmt.Errorf("deployment: secret provider unavailable: %w", domain.ErrAppSecretMissing)
794 }
795 for envKey, name := range svc.Secrets {
796 path := domain.AppSecretPathForID(appID, app, svc.Name, name)
797 if _, err := s.deps.Secrets.GetSecret(ctx, path); err != nil {
798 return fmt.Errorf("deployment: secret %s (env %s) missing: %w", path, envKey, domain.ErrAppSecretMissing)
799 }
800 }
801 return nil
802}
803
804// appSecretID returns the stable internal UUID for secret paths.
805// Empty means legacy name-keyed paths (records predating UUIDs).
806// Used on the execution path where preflight's ownership snapshot is
807// not in scope; deploys are infrequent, one extra store read per
808// service is acceptable.
809func (s *Service) appSecretID(ctx context.Context, app string) (string, error) {
810 ownership, err := s.deps.State.LoadOwnership(ctx, app)
811 if err != nil {
812 return "", err
813 }
814 return ownership.ID, nil
815}
816
817func (s *Service) preflightImage(ctx context.Context, image, digest string) (string, error) {
818 if err := validation.ValidateImageDigest(digest); err != nil {
819 return "", fmt.Errorf("deployment: image %q resolved to invalid digest: %w", image, domain.ErrAppImageUnresolvable)
820 }
821 if err := s.validateImageSource(image, digest); err != nil {
822 return "", err
823 }
824 if s.deps.Registry.Domain == "" {
825 return image, nil
826 }
827 return s.pullImage(ctx, stripImageTag(image)+"@"+digest)
828}
829
830// validateImageSource enforces the installation image policy on a
831// reference before any pull. Both the manifest reference and the pinned
832// digest are checked, so a digest-pinned ref cannot bypass the hostname
833// and port allowlist on deploy or recovery.
834func (s *Service) validateImageSource(image, digest string) error {
835 ref := strings.TrimSpace(image)
836 if digest != "" && !strings.Contains(ref, "@") {
837 ref = stripImageTag(image) + "@" + digest
838 }
839 if err := s.deps.ImagePolicy.ValidateImageSource(ref); err != nil {
840 return fmt.Errorf("deployment: image %q: %w", image, err)
841 }
842 return nil
843}
844
845// checkImageVolumes rejects images with unmapped VOLUME declarations.
846func (s *Service) checkImageVolumes(ctx context.Context, svc domain.AppService, runtimeImage string) error {
847 declared, err := s.deps.Runtime.InspectImageVolumes(ctx, runtimeImage)
848 if err != nil {
849 return fmt.Errorf("deployment: inspect image volumes: %w", err)
850 }
851 if len(declared) == 0 {
852 return nil
853 }
854 mapped := map[string]struct{}{}
855 for _, vol := range svc.Volumes {
856 mapped[vol.Path] = struct{}{}
857 }
858 // Bind destinations also map image-declared volumes: a bind that
859 // mounts over an image VOLUME is an explicit operator mapping.
860 for _, bind := range svc.Binds {
861 mapped[bind.Path] = struct{}{}
862 }
863 for _, path := range declared {
864 if _, ok := mapped[path]; !ok {
865 return fmt.Errorf(
866 "deployment: image declares unmanaged volume %q, map it explicitly: %w",
867 path, domain.ErrAppUnmanagedImageVolume,
868 )
869 }
870 }
871 return nil
872}
873
874// recheckReservations re-validates against the live global table.
875func (s *Service) recheckReservations(ctx context.Context, app string, rev domain.AppDesiredRevision) error {
876 checkpoint, err := s.deps.State.LoadCheckpoint(ctx)
877 if err != nil {
878 return err
879 }
880 reservations := domain.ReservationsFor(rev.Spec)
881 if err := domain.CheckReservations(checkpoint.Reservations, reservations, app); err != nil {
882 return fmt.Errorf("deployment: %w", err)
883 }
884 return nil
885}
886
887// checkResources verifies volumes exist-or-creatable and ownership is consistent.
888// A generated runtime volume that already exists but is NOT recorded in
889// this app's ownership is refused fail-closed: mounting it would
890// implicitly adopt (and possibly modify) foreign data. Volumes recorded
891// in ownership (attached or retained) were created by this app's own
892// deploys and are safe to reuse.
893func (s *Service) checkResources(ctx context.Context, app string, ownership domain.AppOwnership, services []domain.AppService) error {
894 owned := map[string]string{}
895 ownedRuntime := map[string]struct{}{}
896 for _, vol := range ownership.Volumes {
897 owned[vol.Name] = vol.Service
898 ownedRuntime[vol.RuntimeName] = struct{}{}
899 }
900 for _, svc := range services {
901 for _, vol := range svc.Volumes {
902 if owner, ok := owned[vol.Name]; ok && owner != svc.Name {
903 return fmt.Errorf(
904 "deployment: volume %q owned by %q, not %q: %w",
905 vol.Name, owner, svc.Name, domain.ErrAppStateConflict,
906 )
907 }
908 runtimeName := domain.RuntimeVolumeName(app, svc.Name, vol.Name)
909 exists, err := s.deps.Runtime.VolumeExists(ctx, runtimeName)
910 if err != nil {
911 return fmt.Errorf("deployment: check volume %q: %w", runtimeName, err)
912 }
913 if exists {
914 if _, ok := ownedRuntime[runtimeName]; !ok {
915 return fmt.Errorf(
916 "deployment: runtime volume %q already exists but is not owned by app %q: refusing to adopt foreign data: %w",
917 runtimeName, app, domain.ErrAppStateConflict,
918 )
919 }
920 }
921 }
922 }
923 return nil
924}
925
926// ComputeOutcome derives the op outcome over terminal per-service results.
927func ComputeOutcome(results map[string]ServiceResult) string {
928 deployed := 0
929 failed := 0
930 for _, result := range results {
931 switch result.Result {
932 case "deployed":
933 deployed++
934 case "failed":
935 failed++
936 }
937 }
938 switch {
939 case deployed > 0 && failed > 0:
940 return domain.AppOutcomePartial
941 case deployed > 0:
942 return domain.AppOutcomeSuccess
943 default:
944 return domain.AppOutcomeFailed
945 }
946}
947
948// itoa formats integers without extra imports.
949func itoa(n int) string {
950 return fmt.Sprintf("%d", n)
951}