1// Package deployment implements the app deployment engine: preflight,
2// journaled execution, and terminal-result computation.
3package deployment
4
5import (
6 "context"
7 "encoding/hex"
8 "errors"
9 "fmt"
10 "maps"
11 "sort"
12 "strings"
13 "sync"
14 "time"
15
16 "github.com/bnema/zerowrap"
17 "github.com/google/uuid"
18
19 "github.com/bnema/gordon/internal/boundaries/out"
20 "github.com/bnema/gordon/internal/domain"
21 "github.com/bnema/gordon/pkg/validation"
22)
23
24// Deps are the driven adapters owned by the deployment engine.
25type Deps struct {
26 State out.AppState
27 Runtime out.ContainerRuntime
28 Images out.ImageResolver
29 Secrets out.SecretProvider
30 // Registry configures image pulls from the installation registry.
31 // Domain scopes which refs pull with auth; empty disables pulling.
32 Registry RegistryConfig
33 // Networks is the installation network policy applied to every app
34 // workload: the prefix used to derive Gordon-owned network names and
35 // whether those networks block external egress.
36 Networks NetworkConfig
37 // Limits are the installation container resource limits applied on
38 // every create and recovery path. Zero means unlimited.
39 Limits ResourceLimits
40 // ImagePolicy restricts which registries and references may be
41 // resolved, pulled, or run. It is enforced on every path, including
42 // boot, restart, and recovery.
43 ImagePolicy domain.ImageSourcePolicy
44 // Traffic is the serialized HTTP/L4 publish boundary. Rebuilt after
45 // every activation (deploy/start/restart/stop/remove) so the proxy
46 // host index follows ACTIVE; recovery also withdraws one service
47 // fail-closed before touching a workload. Nil in tests that assert
48 // state effects only. Failures are reported to the caller.
49 Traffic out.AppTrafficRefresher
50 // SecretFiles stores file-delivered secrets on the host. Optional:
51 // nil is valid only while no service delivers secrets as files;
52 // preflight refuses such services otherwise.
53 SecretFiles out.SecretFileStore
54 // Guard is checked on every app container config right before the
55 // runtime create. The zero value still refuses host/shared network
56 // namespaces and host paths passed as volume names.
57 Guard domain.RuntimeGuard
58}
59
60// RegistryConfig carries the installation registry identity for pulls.
61type RegistryConfig struct {
62 Domain string
63 PullAddress string
64 Username string
65 Password string
66}
67
68// NetworkConfig carries the installation network policy for app workloads.
69type NetworkConfig struct {
70 // Prefix names the Gordon-owned networks. Empty uses the domain default.
71 Prefix string
72 // Internal blocks external egress from containers attached to those
73 // networks (Docker's Internal flag).
74 Internal bool
75}
76
77// ResourceLimits are the installation container limits applied on every
78// create and recovery path. Zero values mean unlimited.
79type ResourceLimits struct {
80 MemoryBytes int64
81 NanoCPUs int64
82 PidsLimit int64
83}
84
85// Service orchestrates captured-revision deploys.
86type Service struct {
87 deps Deps
88 log zerowrap.Logger
89 probes *ProbeDeps
90
91 // coord serializes workload mutations per app: deploy/start/
92 // restart/stop/remove/boot lock blocking, periodic reconciliation
93 // skips busy apps.
94 coord *appCoordinator
95 // barrier is the process-wide GC barrier. A shared lease covers
96 // every workload mutation from resource selection through durable
97 // protection publication; prune holds the exclusive lease.
98 barrier out.GCBarrier
99 // backoff is the in-memory, generation-scoped recovery budget.
100 backoff *recoveryBackoff
101 // publication holds the in-memory publication inhibition: a
102 // service whose fail-closed withdrawal could not be applied must be
103 // withdrawn again before any later publication.
104 publication publicationInhibition
105 // executions remembers the last observed execution start per
106 // generation, so a native restart is detected without restarting
107 // the workload ourselves.
108 executions executionTracker
109 // bindMu guards bindPolicies. A config reload replaces the whole
110 // map atomically; deploy paths read the current map without holding
111 // the lock.
112 bindMu sync.RWMutex
113 bindPolicies map[string]domain.AppBindPolicy
114 // deviceMu guards devicePolicies. A config reload replaces the whole
115 // map atomically; deploy paths read the current map without holding
116 // the lock.
117 deviceMu sync.RWMutex
118 devicePolicies map[string]domain.AppDevicePolicy
119 // liveMu guards liveOps. liveOps holds the operations this process
120 // owns between the StartDeploy claim and the end of ExecuteDeploy. It
121 // is in-memory only: a fresh Service (boot) starts empty and may
122 // reconcile every persisted non-terminal operation, while a
123 // foreground mutation must never finalize an operation a live
124 // goroutine still owns.
125 liveMu sync.Mutex
126 liveOps map[string]struct{}
127}
128
129// NewService creates the deployment engine. All deps are required;
130// Secrets may be nil only in tests that never reach secret preflight.
131func NewService(deps Deps, log zerowrap.Logger) *Service {
132 if deps.ImagePolicy.InstallationRegistry == "" {
133 deps.ImagePolicy.InstallationRegistry = deps.Registry.Domain
134 }
135 return &Service{
136 deps: deps,
137 log: log,
138 coord: newAppCoordinator(),
139 backoff: newRecoveryBackoff(time.Now),
140 }
141}
142
143// markOperationLive records that this process owns one operation from the
144// StartDeploy claim until ExecuteDeploy finishes. A live operation is the
145// single owner's in-flight work: a foreground reconciliation must not
146// finalize it, or the claim would be settled before the owner can execute it.
147func (s *Service) markOperationLive(app, op string) {
148 if app == "" || op == "" {
149 return
150 }
151 s.liveMu.Lock()
152 defer s.liveMu.Unlock()
153 if s.liveOps == nil {
154 s.liveOps = make(map[string]struct{})
155 }
156 s.liveOps[liveOperationKey(app, op)] = struct{}{}
157}
158
159// clearOperationLive drops the live marker. It is called once ExecuteDeploy
160// has recorded its outcome or left the claim recoverable, so a later
161// reconciliation can converge the operation instead of leaving it in flight.
162func (s *Service) clearOperationLive(app, op string) {
163 s.liveMu.Lock()
164 defer s.liveMu.Unlock()
165 delete(s.liveOps, liveOperationKey(app, op))
166}
167
168// operationLive reports whether this process still owns the operation.
169func (s *Service) operationLive(app, op string) bool {
170 s.liveMu.Lock()
171 defer s.liveMu.Unlock()
172 _, ok := s.liveOps[liveOperationKey(app, op)]
173 return ok
174}
175
176func liveOperationKey(app, op string) string {
177 return app + "\x00" + op
178}
179
180// WithGCBarrier wires the process-wide GC barrier. Every workload
181// mutation holds a shared lease from resource selection through the
182// durable publication of the resource's protection, so prune can never
183// observe a half-acquired resource. Nil is valid for tests that never
184// run prune.
185func (s *Service) WithGCBarrier(barrier out.GCBarrier) *Service {
186 s.barrier = barrier
187 return s
188}
189
190// WithBindPolicies supplies the initial administrative bind policies.
191func (s *Service) WithBindPolicies(policies map[string]domain.AppBindPolicy) *Service {
192 s.SetBindPolicies(policies)
193 return s
194}
195
196// SetBindPolicies atomically replaces the administrative bind policies used
197// to authorize service binds. The map is copied so later caller mutation
198// cannot race an in-flight deploy.
199func (s *Service) SetBindPolicies(policies map[string]domain.AppBindPolicy) {
200 copied := make(map[string]domain.AppBindPolicy, len(policies))
201 for name, policy := range policies {
202 copied[name] = policy
203 }
204 s.bindMu.Lock()
205 s.bindPolicies = copied
206 s.bindMu.Unlock()
207}
208
209// snapshotBindPolicies returns the current policy map. SetBindPolicies never
210// mutates a published map, so the snapshot stays safe after the lock drops.
211func (s *Service) snapshotBindPolicies() map[string]domain.AppBindPolicy {
212 s.bindMu.RLock()
213 defer s.bindMu.RUnlock()
214 return s.bindPolicies
215}
216
217// WithDevicePolicies supplies the initial administrative device policies.
218func (s *Service) WithDevicePolicies(policies map[string]domain.AppDevicePolicy) *Service {
219 s.SetDevicePolicies(policies)
220 return s
221}
222
223// SetDevicePolicies atomically replaces the administrative device policies
224// used to authorize service devices. The map and its slices are copied so
225// later caller mutation cannot race an in-flight deploy.
226func (s *Service) SetDevicePolicies(policies map[string]domain.AppDevicePolicy) {
227 copied := make(map[string]domain.AppDevicePolicy, len(policies))
228 for name, policy := range policies {
229 policy.CDI = append([]string(nil), policy.CDI...)
230 policy.AllowedApps = append([]string(nil), policy.AllowedApps...)
231 policy.AllowedServices = append([]string(nil), policy.AllowedServices...)
232 copied[name] = policy
233 }
234 s.deviceMu.Lock()
235 s.devicePolicies = copied
236 s.deviceMu.Unlock()
237}
238
239// snapshotDevicePolicies returns the current policy map. SetDevicePolicies
240// never mutates a published map, so the snapshot stays safe after the
241// lock drops.
242func (s *Service) snapshotDevicePolicies() map[string]domain.AppDevicePolicy {
243 s.deviceMu.RLock()
244 defer s.deviceMu.RUnlock()
245 return s.devicePolicies
246}
247
248// resolveServiceDevices resolves every declared logical device against the
249// current administrative policies into runtime CDI IDs. Errors name only
250// the app, service, and device: host inventory never appears.
251func (s *Service) resolveServiceDevices(app string, svc domain.AppService) ([]string, error) {
252 if len(svc.Devices) == 0 {
253 return nil, nil
254 }
255 ids, err := domain.ResolveAppDevices(app, svc.Name, svc.Devices, s.snapshotDevicePolicies())
256 if err != nil {
257 return nil, fmt.Errorf("deployment: %w", err)
258 }
259 return ids, nil
260}
261
262// resolveServiceBinds resolves every declared bind against the current
263// administrative policies into ephemeral runtime binds. Errors name only the
264// app, service, and mount: host source paths never appear.
265func (s *Service) resolveServiceBinds(app string, svc domain.AppService) ([]domain.ContainerBind, error) {
266 if len(svc.Binds) == 0 {
267 return nil, nil
268 }
269 policies := s.snapshotBindPolicies()
270 resolved := make([]domain.ContainerBind, 0, len(svc.Binds))
271 for _, bind := range svc.Binds {
272 policy, ok := policies[bind.Name]
273 if !ok {
274 return nil, fmt.Errorf("deployment: app %q service %q mount %q: no administrative mount policy configured: %w", app, svc.Name, bind.Name, domain.ErrBindPolicy)
275 }
276 r, err := policy.ResolveAppBind(app, svc.Name, bind)
277 if err != nil {
278 return nil, fmt.Errorf("deployment: app %q service %q mount %q: refused by administrative mount policy: %w", app, svc.Name, bind.Name, domain.ErrBindPolicy)
279 }
280 resolved = append(resolved, domain.ContainerBind(r))
281 }
282 // Apply the runtime guard here so a bind overlapping a protected
283 // Gordon path fails at preflight and before the serving generation is
284 // retired, not only at create time. createContainer keeps the full
285 // guard as the final check.
286 if err := s.deps.Guard.Check(&domain.ContainerConfig{Binds: resolved}); err != nil {
287 return nil, fmt.Errorf("deployment: app %q service %q: %w", app, svc.Name, err)
288 }
289 return resolved, nil
290}
291
292// acquireAppContext takes the GC shared lease first and the per-app
293// coordinator second, following the documented lock order
294// (GC barrier → per-app coordinator → registry lock → bbolt
295// transaction). The returned release drops both, coordinator first.
296func (s *Service) acquireAppContext(ctx context.Context, app string) (func(), error) {
297 lease, err := s.acquireSharedGC(ctx)
298 if err != nil {
299 return nil, err
300 }
301 release, err := s.coord.acquire(ctx, app)
302 if err != nil {
303 lease.Release()
304 return nil, err
305 }
306 return func() {
307 release()
308 lease.Release()
309 }, nil
310}
311
312// tryAcquireAppContext takes the GC shared lease, then tries the per-app
313// coordinator without blocking. ok is false when the app is busy; the
314// caller skips it. Periodic reconciliation still waits for an in-flight
315// prune, because a half-planned snapshot is exactly what must not be
316// raced.
317func (s *Service) tryAcquireAppContext(ctx context.Context, app string) (func(), bool) {
318 lease, err := s.acquireSharedGC(ctx)
319 if err != nil {
320 return nil, false
321 }
322 release, ok := s.coord.tryAcquire(app)
323 if !ok {
324 lease.Release()
325 return nil, false
326 }
327 return func() {
328 release()
329 lease.Release()
330 }, true
331}
332
333// acquireSharedGC takes a shared GC lease, or a no-op lease when no
334// barrier is wired.
335func (s *Service) acquireSharedGC(ctx context.Context) (out.GCLease, error) {
336 if s.barrier == nil {
337 return noopGCLease{}, nil
338 }
339 return s.barrier.AcquireShared(ctx)
340}
341
342// noopGCLease is the lease used when no barrier is wired.
343type noopGCLease struct{}
344
345func (noopGCLease) Release() {}
346
347// WithProbeDeps overrides readiness probing (tests inject a mock-backed
348// ProbeDeps via NewProbeDeps; production leaves it nil for the live
349// runtime adapter).
350func (s *Service) WithProbeDeps(probes ProbeDeps) *Service {
351 s.probes = &probes
352 return s
353}
354
355// DeployInput selects the revision and service scope.
356type DeployInput struct {
357 App string
358 Revision string
359 Service string
360 Op string
361}
362
363// DeployResult carries terminal per-service results and cleanup warnings.
364type DeployResult struct {
365 Op string
366 App string
367 Revision string
368 Outcome string
369 Services map[string]ServiceResult
370 CleanupWarnings []CleanupWarning
371 // Removed names services retired because the new revision no longer
372 // declares them.
373 Removed []string
374}
375
376// ServiceResultUnchanged marks a deploy step that kept the running
377// container because its image, spec, and environment were already current.
378const ServiceResultUnchanged = domain.AppServiceUnchanged
379
380// ServiceResult is one service's terminal deployment result.
381type ServiceResult struct {
382 Result string
383 EffectiveRevision string
384 Before string
385 After string
386 RestartUnsafe bool
387 Error string
388 // BackendBinds carries the created container's TCP loopback publishes
389 // (container port -> 127.0.0.1 host port) for the active record.
390 BackendBinds map[int]int
391 // UDPBackendBinds carries the created container's UDP loopback
392 // publishes for the active record. Nil when the service declares
393 // no UDP interface.
394 UDPBackendBinds map[int]int
395 // CleanupWarnings are bounded leftovers of this service's terminal
396 // path: a candidate that could not be removed, or a backend claim
397 // that could not be released.
398 CleanupWarnings []CleanupWarning
399 // Diagnostics is bounded, redacted failure output of the failed
400 // candidate. It is never embedded in Error and is persisted to the
401 // journal for logs-scoped retrieval only.
402 Diagnostics []string
403}
404
405// CleanupWarning records post-publication leftovers for operator action.
406type CleanupWarning struct {
407 Service string
408 Leftover string
409 Detail string
410}
411
412// pinnedService binds a service spec to its resolved digest plus the
413// captured revision's app-wide public env (injected into every service
414// alongside that service's own resolved secrets). appEnv is a copy owned
415// by this pin: never mutated, never persisted with secret values.
416type pinnedService struct {
417 name string
418 spec domain.AppService
419 digest string
420 runtimeImage string
421 appEnv map[string]string
422 appNetworks []domain.AppSharedNetwork
423 // sharedNetworks are the declared shared-network memberships this
424 // service joins. The private incarnation network is derived from the
425 // app UUID at create time so recovery and deploy agree.
426 sharedNetworks []domain.AppSharedNetwork
427}
428
429// newOpID allocates a time-ordered op- identifier (uuid v7).
430func newOpID() string {
431 id, err := uuid.NewV7()
432 if err != nil {
433 id = uuid.New()
434 }
435 var buf [32]byte
436 hex.Encode(buf[:], id[:])
437 return "op-" + string(buf[:])
438}
439
440// claimDeploymentLocked is the first phase of a deploy: it resolves the
441// requested revision, enforces the targeted-deploy convergence rules, and
442// claims the request key, persisting the non-terminal journal. It performs
443// no image pull and no workload mutation, so a crash after it leaves a
444// durable claim that reconciliation can converge. owned is true only for the
445// single claimer of the key; a replay returns the stored journal instead.
446// The caller holds the app lock and has already recovered the store.
447func (s *Service) claimDeploymentLocked(ctx context.Context, input DeployInput) (domain.AppDesiredRevision, domain.AppOperation, bool, error) {
448 rev, err := s.resolveRevision(ctx, input)
449 if err != nil {
450 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
451 }
452 if input.Service != "" {
453 if err := s.checkConverged(ctx, input.App, input.Service, rev); err != nil {
454 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
455 }
456 }
457
458 op := domain.AppOperation{
459 Kind: "deploy",
460 App: input.App,
461 InputRevision: rev.Revision,
462 StartedAt: time.Now().UTC(),
463 Request: domain.AppOperationRequestFor("deploy", input.App, input.Revision, input.Service),
464 }
465 op, owned, err := s.claimOperation(ctx, input.Op, op, []domain.AppOperationStep{{ID: "preflight", State: domain.AppStepPending}})
466 if err != nil {
467 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
468 }
469 return rev, op, owned, nil
470}
471
472// pinPreflightLocked is the second preflight phase: it runs the image
473// pull/pin and resource gates of an already claimed operation and records the
474// pinned table (or the terminal preflight failure) in the journal. The
475// caller holds the app lock and owns the claimed, non-terminal operation.
476func (s *Service) pinPreflightLocked(ctx context.Context, app, onlyService string, rev domain.AppDesiredRevision, op *domain.AppOperation) ([]pinnedService, error) {
477 log := zerowrap.FromCtx(ctx)
478
479 pinned, err := s.preflightServices(ctx, app, rev, onlyService)
480 if err != nil {
481 op.Steps[0] = domain.AppOperationStep{ID: "preflight", State: domain.AppStepFailed, Error: err.Error()}
482 op.Outcome = domain.AppOutcomeFailed
483 if saveErr := s.deps.State.SaveOperation(ctx, *op); saveErr != nil {
484 log.Warn().Err(saveErr).Msg("deployment: failed to record preflight failure")
485 }
486 return nil, err
487 }
488 op.Steps[0] = domain.AppOperationStep{ID: "preflight", State: domain.AppStepSucceeded}
489 // A resumed claim can carry the service-step plan of an earlier attempt,
490 // including candidates it recorded. Converge those candidates before
491 // replacing the plan: an unreconciled leftover may still be running, and
492 // the replacement this execution creates must never overlap it.
493 if err := s.convergeResumedCandidates(ctx, app, op, log); err != nil {
494 return nil, err
495 }
496 // Replace the persisted plan instead of appending: the preflight step plus
497 // one step per pinned service, so runServiceStep's index contract matches a
498 // normalized layout and no earlier step is overwritten or left pending.
499 plan := make([]domain.AppOperationStep, 1, len(pinned)+1)
500 plan[0] = op.Steps[0]
501 for _, p := range pinned {
502 plan = append(plan, domain.AppOperationStep{
503 ID: "service." + p.name + ".replace",
504 State: domain.AppStepPending,
505 Service: p.name,
506 Digest: p.digest,
507 Image: p.runtimeImage,
508 })
509 }
510 op.Steps = plan
511 if err := s.deps.State.SaveOperation(ctx, *op); err != nil {
512 return nil, fmt.Errorf("deployment: persist pinned table: %w", err)
513 }
514 log.Info().Str("op", op.Op).Str("revision", rev.Revision).Int("services", len(pinned)).Msg("deployment: preflight passed")
515 return pinned, nil
516}
517
518// convergeResumedCandidates converges any candidate an earlier attempt of a
519// resumed claim recorded, so replacing its service-step plan cannot drop a
520// leftover that may still be running. A fresh claim has only its preflight
521// step and does no work here.
522func (s *Service) convergeResumedCandidates(ctx context.Context, app string, op *domain.AppOperation, log zerowrap.Logger) error {
523 needsActive := false
524 for _, step := range op.Steps {
525 if reconcileNeedsContainer(step, false) {
526 needsActive = true
527 break
528 }
529 }
530 if !needsActive {
531 return nil
532 }
533 active, _, err := s.deps.State.LoadActive(ctx, app)
534 if err != nil {
535 return fmt.Errorf("deployment: load active for resumed claim: %w", err)
536 }
537 if _, err := s.convergeInterruptedSteps(ctx, app, op, active, false, log); err != nil {
538 return err
539 }
540 return nil
541}
542
543// claimOperation is the single claim point before any mutation effect.
544// A keyed request is claimed atomically: the first claim persists the
545// in-flight journal and reports owned=true; a repeat returns the stored
546// journal and reports owned=false. A key that already answered a
547// different request fails with ErrAppStateConflict, and a key for a name
548// with no app identity fails with ErrAppNotFound without writing
549// anything. An unkeyed request (internal recovery paths) starts a fresh
550// journal under a generated id.
551func (s *Service) claimOperation(ctx context.Context, key string, op domain.AppOperation, steps []domain.AppOperationStep) (domain.AppOperation, bool, error) {
552 op.Steps = steps
553 // A journal is never opened while a different operation of this app is
554 // still unfinished: that predecessor must be reconciled first, or this
555 // claim would mask it and its leftover generation could stay untracked.
556 latest, found, err := s.deps.State.LoadLatestOperation(ctx, op.App)
557 if err != nil {
558 return domain.AppOperation{}, false, fmt.Errorf("deployment: load latest operation: %w", err)
559 }
560 if found && !latest.Terminal() && latest.Op != key {
561 return domain.AppOperation{}, false, fmt.Errorf(
562 "deployment: app %q has unfinished operation %s: %w", op.App, latest.Op, domain.ErrAppStateConflict)
563 }
564 if key == "" {
565 op.Op = newOpID()
566 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
567 return domain.AppOperation{}, false, fmt.Errorf("deployment: persist journal before effects: %w", err)
568 }
569 return op, true, nil
570 }
571 op.Op = key
572 existing, claimed, err := s.deps.State.ClaimOperation(ctx, op)
573 if err != nil {
574 return domain.AppOperation{}, false, err
575 }
576 if !claimed {
577 return existing, false, nil
578 }
579 return op, true, nil
580}
581
582// replayError reports a replayed key whose journal never reached a
583// terminal outcome: the operation is still in flight or was interrupted,
584// so its effects must not run again.
585func replayError(op domain.AppOperation) error {
586 if !op.Terminal() {
587 return fmt.Errorf("deployment: operation %s has not reached a terminal outcome: %w", op.Op, domain.ErrAppStateConflict)
588 }
589 if op.Outcome != domain.AppOutcomeSuccess {
590 return fmt.Errorf("deployment: operation %s previously ended with outcome %s: %w", op.Op, op.Outcome, domain.ErrAppStateConflict)
591 }
592 return nil
593}
594
595// resolveRevision loads the captured revision (default: current desired).
596// A revision that cannot be resolved on a name with no app identity at
597// all reports ErrAppNotFound: the failure is the missing app, not the
598// missing revision.
599func (s *Service) resolveRevision(ctx context.Context, input DeployInput) (domain.AppDesiredRevision, error) {
600 if input.Revision != "" {
601 rev, err := s.deps.State.LoadRevision(ctx, input.App, input.Revision)
602 if err != nil && errors.Is(err, domain.ErrAppRevisionNotFound) {
603 if knownErr := s.requireKnownApp(ctx, input.App); knownErr != nil {
604 return domain.AppDesiredRevision{}, knownErr
605 }
606 }
607 return rev, err
608 }
609 rev, ok, err := s.deps.State.LoadDesired(ctx, input.App)
610 if err != nil {
611 return domain.AppDesiredRevision{}, err
612 }
613 if !ok {
614 if knownErr := s.requireKnownApp(ctx, input.App); knownErr != nil {
615 return domain.AppDesiredRevision{}, knownErr
616 }
617 return domain.AppDesiredRevision{}, fmt.Errorf("deployment: app %q has no desired state: %w", input.App, domain.ErrAppRevisionNotFound)
618 }
619 return rev, nil
620}
621
622// requireKnownApp refuses a mutation of a name with no live app identity.
623func (s *Service) requireKnownApp(ctx context.Context, app string) error {
624 exists, err := s.deps.State.AppExists(ctx, app)
625 if err != nil {
626 return fmt.Errorf("deployment: app existence: %w", err)
627 }
628 if !exists {
629 return fmt.Errorf("deployment: app %q does not exist: %w", app, domain.ErrAppNotFound)
630 }
631 return nil
632}
633
634// checkConverged permits a targeted deploy when every desired/effective
635// difference belongs to that service. App-wide and other-service changes
636// must be deployed together so ACTIVE never combines incompatible specs.
637func (s *Service) checkConverged(ctx context.Context, app, service string, rev domain.AppDesiredRevision) error {
638 active, ok, err := s.deps.State.LoadActive(ctx, app)
639 if err != nil {
640 return err
641 }
642 if !ok {
643 return fmt.Errorf("deployment: app %q was never deployed, targeted deploy refused: %w", app, domain.ErrAppStateConflict)
644 }
645 if active.Converged && active.ConvergedRevision == rev.Revision {
646 return nil
647 }
648 diff := domain.DiffAppSpec(rev.Spec, effectiveAppSpec(active))
649 prefix := "service/" + service + "/"
650 if len(diff.Added) > 0 {
651 return targetedDivergenceError(rev.Revision, diff.Added[0]+" (added)")
652 }
653 if len(diff.Removed) > 0 {
654 return targetedDivergenceError(rev.Revision, diff.Removed[0]+" (removed)")
655 }
656 for _, path := range diff.Changed {
657 if !strings.HasPrefix(path, prefix) {
658 return targetedDivergenceError(rev.Revision, path)
659 }
660 }
661 return nil
662}
663
664func effectiveAppSpec(active domain.AppActive) domain.AppSpec {
665 spec := domain.AppSpec{Name: active.App, Networks: append([]domain.AppSharedNetwork(nil), active.Networks...)}
666 for name, service := range active.Services {
667 effective := service.Spec
668 effective.Name = name
669 spec.Services = append(spec.Services, effective)
670 }
671 return spec
672}
673
674func targetedDivergenceError(revision, path string) error {
675 return fmt.Errorf("%w: service-targeted deploy refused, desired %s also changes %s", domain.ErrAppStateConflict, revision, path)
676}
677
678// selectPreflightServices copies and optionally narrows a revision's services.
679func selectPreflightServices(rev domain.AppDesiredRevision, onlyService string) ([]domain.AppService, error) {
680 services := append([]domain.AppService(nil), rev.Spec.Services...)
681 if onlyService == "" {
682 return services, nil
683 }
684 for _, svc := range services {
685 if svc.Name == onlyService {
686 return []domain.AppService{svc}, nil
687 }
688 }
689 return nil, fmt.Errorf("deployment: service %q not in revision %s: %w", onlyService, rev.Revision, domain.ErrAppStateConflict)
690}
691
692// preflightServices runs the preflight gates in order. No mutation.
693func (s *Service) preflightServices(ctx context.Context, app string, rev domain.AppDesiredRevision, onlyService string) ([]pinnedService, error) {
694 services, err := selectPreflightServices(rev, onlyService)
695 if err != nil {
696 return nil, err
697 }
698 sort.Slice(services, func(i, j int) bool { return services[i].Name < services[j].Name })
699
700 var pinned []pinnedService
701 ownership, err := s.deps.State.LoadOwnership(ctx, app)
702 if err != nil {
703 return nil, err
704 }
705 // Defensive re-check: stored revisions predate or bypass apply
706 // validation; an overlap must fail preflight before any mutation.
707 if err := rev.Spec.CheckEnvSecretCollisions(); err != nil {
708 return nil, err
709 }
710 appEnv := maps.Clone(rev.Spec.Env)
711 if appEnv == nil {
712 appEnv = map[string]string{}
713 }
714 for _, svc := range services {
715 digest, err := s.deps.Images.ResolveDigest(ctx, svc.Image)
716 if err != nil {
717 return nil, fmt.Errorf("deployment: image %q unresolvable: %w", svc.Image, domain.ErrAppImageUnresolvable)
718 }
719 if err := s.checkSecrets(ctx, app, ownership.ID, svc); err != nil {
720 return nil, err
721 }
722 runtimeImage, err := s.preflightImage(ctx, svc.Image, digest)
723 if err != nil {
724 return nil, err
725 }
726 if err := s.checkImageVolumes(ctx, svc, runtimeImage); err != nil {
727 return nil, err
728 }
729 // Bind resolution stays a preflight gate: an unresolvable bind
730 // must fail before any workload mutation. The successful values
731 // are discarded because createAndStart re-resolves against the
732 // current policy immediately before the runtime mutation.
733 if _, err := s.resolveServiceBinds(app, svc); err != nil {
734 return nil, err
735 }
736 if err := s.preflightServiceDevices(ctx, app, svc, pinned); err != nil {
737 return nil, err
738 }
739 pinned = append(pinned, pinnedService{
740 name: svc.Name, spec: svc, digest: digest, runtimeImage: runtimeImage, appEnv: appEnv,
741 appNetworks: append([]domain.AppSharedNetwork(nil), rev.Spec.Networks...),
742 sharedNetworks: domain.AppServiceSharedNetworks(rev.Spec, svc.Name),
743 })
744 }
745 if err := s.recheckReservations(ctx, app, rev); err != nil {
746 return nil, err
747 }
748 if err := s.checkResources(ctx, app, ownership, services); err != nil {
749 return nil, err
750 }
751 return pinned, nil
752}
753
754// preflightServiceDevices runs the device authorization and engine
755// capability gates for one service. Split from preflightServices to keep
756// its complexity within budget.
757func (s *Service) preflightServiceDevices(ctx context.Context, app string, svc domain.AppService, pinned []pinnedService) error {
758 if _, err := s.resolveServiceDevices(app, svc); err != nil {
759 return err
760 }
761 // Engine capability is a knowable-before-mutation failure: a
762 // device-bearing revision on an unsupported engine must fail here,
763 // before withdrawal retires the serving generation. The check runs
764 // once per revision, on the first device-bearing service;
765 // CreateContainer keeps the same gate as defense.
766 if len(svc.Devices) > 0 && !devicesEngineChecked(pinned) {
767 if s.deps.Runtime == nil {
768 return fmt.Errorf("deployment: runtime unavailable: %w", domain.ErrRuntimeUnsupported)
769 }
770 // Sanitize the adapter cause: version-probe failures may embed
771 // the daemon endpoint, which must not reach journals or API
772 // responses. Cancellation and deadlines are the exception: they
773 // stay recognizable so aborted deploys are not misread as
774 // incapable engines.
775 if err := s.deps.Runtime.SupportsCDIDevices(ctx); err != nil {
776 return sanitizedEngineProbeError(svc.Name, err)
777 }
778 }
779 return nil
780}
781
782// sanitizedEngineProbeError maps an engine device-capability probe failure
783// onto the caller-visible error. Cancellation and deadlines survive so
784// shutdown and timeout handling keep working; every other cause collapses
785// to ErrRuntimeUnsupported alone, dropping the adapter text that may embed
786// the daemon endpoint.
787func sanitizedEngineProbeError(service string, err error) error {
788 switch {
789 case errors.Is(err, context.Canceled):
790 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, context.Canceled)
791 case errors.Is(err, context.DeadlineExceeded):
792 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, context.DeadlineExceeded)
793 default:
794 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, domain.ErrRuntimeUnsupported)
795 }
796}
797
798// devicesEngineChecked reports whether an earlier pinned service already
799// triggered the engine capability probe for this revision.
800func devicesEngineChecked(pinned []pinnedService) bool {
801 for _, p := range pinned {
802 if len(p.spec.Devices) > 0 {
803 return true
804 }
805 }
806 return false
807}
808
809// checkSecrets reads every required secret path once. Values stay in
810// memory for container creation only; nothing is written anywhere.
811// Paths are UUID-keyed so a removed app's secrets are never adopted
812// by a new app reusing the name.
813func (s *Service) checkSecrets(ctx context.Context, app, appID string, svc domain.AppService) error {
814 if len(svc.SecretFiles) > 0 && s.deps.SecretFiles == nil {
815 return fmt.Errorf("deployment: service %q delivers secrets as files but no secret file store is configured", svc.Name)
816 }
817 if s.deps.Secrets == nil {
818 return fmt.Errorf("deployment: secret provider unavailable: %w", domain.ErrAppSecretMissing)
819 }
820 for envKey, name := range svc.Secrets {
821 path := domain.AppSecretPathForID(appID, app, svc.Name, name)
822 if _, err := s.deps.Secrets.GetSecret(ctx, path); err != nil {
823 return fmt.Errorf("deployment: secret %s (env %s) missing: %w", path, envKey, domain.ErrAppSecretMissing)
824 }
825 }
826 return nil
827}
828
829// appSecretID returns the stable internal UUID for secret paths.
830// Empty means legacy name-keyed paths (records predating UUIDs).
831// Used on the execution path where preflight's ownership snapshot is
832// not in scope; deploys are infrequent, one extra store read per
833// service is acceptable.
834func (s *Service) appSecretID(ctx context.Context, app string) (string, error) {
835 ownership, err := s.deps.State.LoadOwnership(ctx, app)
836 if err != nil {
837 return "", err
838 }
839 return ownership.ID, nil
840}
841
842func (s *Service) preflightImage(ctx context.Context, image, digest string) (string, error) {
843 if err := validation.ValidateImageDigest(digest); err != nil {
844 return "", fmt.Errorf("deployment: image %q resolved to invalid digest: %w", image, domain.ErrAppImageUnresolvable)
845 }
846 if err := s.validateImageSource(image, digest); err != nil {
847 return "", err
848 }
849 if s.deps.Registry.Domain == "" {
850 return image, nil
851 }
852 return s.pullImage(ctx, stripImageTag(image)+"@"+digest)
853}
854
855// validateImageSource enforces the installation image policy on a
856// reference before any pull. Both the manifest reference and the pinned
857// digest are checked, so a digest-pinned ref cannot bypass the hostname
858// and port allowlist on deploy or recovery.
859func (s *Service) validateImageSource(image, digest string) error {
860 ref := strings.TrimSpace(image)
861 if digest != "" && !strings.Contains(ref, "@") {
862 ref = stripImageTag(image) + "@" + digest
863 }
864 if err := s.deps.ImagePolicy.ValidateImageSource(ref); err != nil {
865 return fmt.Errorf("deployment: image %q: %w", image, err)
866 }
867 return nil
868}
869
870// checkImageVolumes rejects images with unmapped VOLUME declarations.
871func (s *Service) checkImageVolumes(ctx context.Context, svc domain.AppService, runtimeImage string) error {
872 declared, err := s.deps.Runtime.InspectImageVolumes(ctx, runtimeImage)
873 if err != nil {
874 return fmt.Errorf("deployment: inspect image volumes: %w", err)
875 }
876 if len(declared) == 0 {
877 return nil
878 }
879 mapped := map[string]struct{}{}
880 for _, vol := range svc.Volumes {
881 mapped[vol.Path] = struct{}{}
882 }
883 // Bind destinations also map image-declared volumes: a bind that
884 // mounts over an image VOLUME is an explicit operator mapping.
885 for _, bind := range svc.Binds {
886 mapped[bind.Path] = struct{}{}
887 }
888 for _, path := range declared {
889 if _, ok := mapped[path]; !ok {
890 return fmt.Errorf(
891 "deployment: image declares unmanaged volume %q, map it explicitly: %w",
892 path, domain.ErrAppUnmanagedImageVolume,
893 )
894 }
895 }
896 return nil
897}
898
899// recheckReservations re-validates against the live global table.
900func (s *Service) recheckReservations(ctx context.Context, app string, rev domain.AppDesiredRevision) error {
901 checkpoint, err := s.deps.State.LoadCheckpoint(ctx)
902 if err != nil {
903 return err
904 }
905 reservations := domain.ReservationsFor(rev.Spec)
906 if err := domain.CheckReservations(checkpoint.Reservations, reservations, app); err != nil {
907 return fmt.Errorf("deployment: %w", err)
908 }
909 return nil
910}
911
912// checkResources verifies volumes exist-or-creatable and ownership is consistent.
913// A generated runtime volume that already exists but is NOT recorded in
914// this app's ownership is refused fail-closed: mounting it would
915// implicitly adopt (and possibly modify) foreign data. Volumes recorded
916// in ownership (attached or retained) were created by this app's own
917// deploys and are safe to reuse.
918func (s *Service) checkResources(ctx context.Context, app string, ownership domain.AppOwnership, services []domain.AppService) error {
919 owned := map[string]string{}
920 ownedRuntime := map[string]struct{}{}
921 for _, vol := range ownership.Volumes {
922 owned[vol.Name] = vol.Service
923 ownedRuntime[vol.RuntimeName] = struct{}{}
924 }
925 for _, svc := range services {
926 for _, vol := range svc.Volumes {
927 if owner, ok := owned[vol.Name]; ok && owner != svc.Name {
928 return fmt.Errorf(
929 "deployment: volume %q owned by %q, not %q: %w",
930 vol.Name, owner, svc.Name, domain.ErrAppStateConflict,
931 )
932 }
933 runtimeName := domain.RuntimeVolumeName(app, svc.Name, vol.Name)
934 exists, err := s.deps.Runtime.VolumeExists(ctx, runtimeName)
935 if err != nil {
936 return fmt.Errorf("deployment: check volume %q: %w", runtimeName, err)
937 }
938 if exists {
939 if _, ok := ownedRuntime[runtimeName]; !ok {
940 return fmt.Errorf(
941 "deployment: runtime volume %q already exists but is not owned by app %q: refusing to adopt foreign data: %w",
942 runtimeName, app, domain.ErrAppStateConflict,
943 )
944 }
945 }
946 }
947 }
948 return nil
949}
950
951// ComputeOutcome derives the op outcome over terminal per-service results.
952func ComputeOutcome(results map[string]ServiceResult) string {
953 deployed := 0
954 failed := 0
955 for _, result := range results {
956 switch result.Result {
957 case "deployed", ServiceResultUnchanged, domain.AppServiceRemoved:
958 deployed++
959 case "failed":
960 failed++
961 }
962 }
963 switch {
964 case deployed > 0 && failed > 0:
965 return domain.AppOutcomePartial
966 case deployed > 0:
967 return domain.AppOutcomeSuccess
968 default:
969 return domain.AppOutcomeFailed
970 }
971}
972
973// itoa formats integers without extra imports.
974func itoa(n int) string {
975 return fmt.Sprintf("%d", n)
976}