1// Package deployment implements the app deployment engine: preflight,
2// journaled execution, and terminal-result computation.
3package deployment
4
5import (
6 "context"
7 "encoding/hex"
8 "errors"
9 "fmt"
10 "maps"
11 "sort"
12 "strings"
13 "sync"
14 "time"
15
16 "github.com/bnema/zerowrap"
17 "github.com/google/uuid"
18
19 "github.com/bnema/gordon/internal/boundaries/out"
20 "github.com/bnema/gordon/internal/domain"
21 "github.com/bnema/gordon/pkg/validation"
22)
23
24// Deps are the driven adapters owned by the deployment engine.
25type Deps struct {
26 State out.AppState
27 Runtime out.ContainerRuntime
28 Images out.ImageResolver
29 Secrets out.SecretProvider
30 // Registry configures image pulls from the installation registry.
31 // Domain scopes which refs pull with auth; empty disables pulling.
32 Registry RegistryConfig
33 // Networks is the installation network policy applied to every app
34 // workload: the prefix used to derive Gordon-owned network names and
35 // whether those networks block external egress.
36 Networks NetworkConfig
37 // Limits are the installation container resource limits applied on
38 // every create and recovery path. Zero means unlimited.
39 Limits ResourceLimits
40 // ImagePolicy restricts which registries and references may be
41 // resolved, pulled, or run. It is enforced on every path, including
42 // boot, restart, and recovery.
43 ImagePolicy domain.ImageSourcePolicy
44 // Traffic is the serialized HTTP/L4 publish boundary. Rebuilt after
45 // every activation (deploy/start/restart/stop/remove) so the proxy
46 // host index follows ACTIVE; recovery also withdraws one service
47 // fail-closed before touching a workload. Nil in tests that assert
48 // state effects only. Failures are reported to the caller.
49 Traffic out.AppTrafficRefresher
50 // SecretFiles stores file-delivered secrets on the host. Optional:
51 // nil is valid only while no service delivers secrets as files;
52 // preflight refuses such services otherwise.
53 SecretFiles out.SecretFileStore
54 // Guard is checked on every app container config right before the
55 // runtime create. The zero value still refuses host/shared network
56 // namespaces and host paths passed as volume names.
57 Guard domain.RuntimeGuard
58}
59
60// RegistryConfig carries the installation registry identity for pulls.
61type RegistryConfig struct {
62 Domain string
63 PullAddress string
64 Username string
65 Password string
66}
67
68// NetworkConfig carries the installation network policy for app workloads.
69type NetworkConfig struct {
70 // Prefix names the Gordon-owned networks. Empty uses the domain default.
71 Prefix string
72 // Internal blocks external egress from containers attached to those
73 // networks (Docker's Internal flag).
74 Internal bool
75}
76
77// ResourceLimits are the installation container limits applied on every
78// create and recovery path. Zero values mean unlimited.
79type ResourceLimits struct {
80 MemoryBytes int64
81 NanoCPUs int64
82 PidsLimit int64
83}
84
85// Service orchestrates captured-revision deploys.
86type Service struct {
87 deps Deps
88 log zerowrap.Logger
89 probes *ProbeDeps
90
91 // coord serializes workload mutations per app: deploy/start/
92 // restart/stop/remove/boot lock blocking, periodic reconciliation
93 // skips busy apps.
94 coord *appCoordinator
95 // barrier is the process-wide GC barrier. A shared lease covers
96 // every workload mutation from resource selection through durable
97 // protection publication; prune holds the exclusive lease.
98 barrier out.GCBarrier
99 // backoff is the in-memory, generation-scoped recovery budget.
100 backoff *recoveryBackoff
101 // publication holds the in-memory publication inhibition: a
102 // service whose fail-closed withdrawal could not be applied must be
103 // withdrawn again before any later publication.
104 publication publicationInhibition
105 // executions remembers the last observed execution start per
106 // generation, so a native restart is detected without restarting
107 // the workload ourselves.
108 executions executionTracker
109 // bindMu guards bindPolicies. A config reload replaces the whole
110 // map atomically; deploy paths read the current map without holding
111 // the lock.
112 bindMu sync.RWMutex
113 bindPolicies map[string]domain.AppBindPolicy
114 // deviceMu guards devicePolicies. A config reload replaces the whole
115 // map atomically; deploy paths read the current map without holding
116 // the lock.
117 deviceMu sync.RWMutex
118 devicePolicies map[string]domain.AppDevicePolicy
119 // liveMu guards liveOps. liveOps holds the operations this process
120 // owns between the StartDeploy claim and the end of ExecuteDeploy. It
121 // is in-memory only: a fresh Service (boot) starts empty and may
122 // reconcile every persisted non-terminal operation, while a
123 // foreground mutation must never finalize an operation a live
124 // goroutine still owns.
125 liveMu sync.Mutex
126 liveOps map[string]struct{}
127}
128
129// NewService creates the deployment engine. All deps are required;
130// Secrets may be nil only in tests that never reach secret preflight.
131func NewService(deps Deps, log zerowrap.Logger) *Service {
132 if deps.ImagePolicy.InstallationRegistry == "" {
133 deps.ImagePolicy.InstallationRegistry = deps.Registry.Domain
134 }
135 return &Service{
136 deps: deps,
137 log: log,
138 coord: newAppCoordinator(),
139 backoff: newRecoveryBackoff(time.Now),
140 }
141}
142
143// markOperationLive records that this process owns one operation from the
144// StartDeploy claim until ExecuteDeploy finishes. A live operation is the
145// single owner's in-flight work: a foreground reconciliation must not
146// finalize it, or the claim would be settled before the owner can execute it.
147func (s *Service) markOperationLive(app, op string) {
148 if app == "" || op == "" {
149 return
150 }
151 s.liveMu.Lock()
152 defer s.liveMu.Unlock()
153 if s.liveOps == nil {
154 s.liveOps = make(map[string]struct{})
155 }
156 s.liveOps[liveOperationKey(app, op)] = struct{}{}
157}
158
159// clearOperationLive drops the live marker. It is called once ExecuteDeploy
160// has recorded its outcome or left the claim recoverable, so a later
161// reconciliation can converge the operation instead of leaving it in flight.
162func (s *Service) clearOperationLive(app, op string) {
163 s.liveMu.Lock()
164 defer s.liveMu.Unlock()
165 delete(s.liveOps, liveOperationKey(app, op))
166}
167
168// operationLive reports whether this process still owns the operation.
169func (s *Service) operationLive(app, op string) bool {
170 s.liveMu.Lock()
171 defer s.liveMu.Unlock()
172 _, ok := s.liveOps[liveOperationKey(app, op)]
173 return ok
174}
175
176func liveOperationKey(app, op string) string {
177 return app + "\x00" + op
178}
179
180// WithGCBarrier wires the process-wide GC barrier. Every workload
181// mutation holds a shared lease from resource selection through the
182// durable publication of the resource's protection, so prune can never
183// observe a half-acquired resource. Nil is valid for tests that never
184// run prune.
185func (s *Service) WithGCBarrier(barrier out.GCBarrier) *Service {
186 s.barrier = barrier
187 return s
188}
189
190// WithBindPolicies supplies the initial administrative bind policies.
191func (s *Service) WithBindPolicies(policies map[string]domain.AppBindPolicy) *Service {
192 s.SetBindPolicies(policies)
193 return s
194}
195
196// SetBindPolicies atomically replaces the administrative bind policies used
197// to authorize service binds. The map is copied so later caller mutation
198// cannot race an in-flight deploy.
199func (s *Service) SetBindPolicies(policies map[string]domain.AppBindPolicy) {
200 copied := make(map[string]domain.AppBindPolicy, len(policies))
201 for name, policy := range policies {
202 copied[name] = policy
203 }
204 s.bindMu.Lock()
205 s.bindPolicies = copied
206 s.bindMu.Unlock()
207}
208
209// snapshotBindPolicies returns the current policy map. SetBindPolicies never
210// mutates a published map, so the snapshot stays safe after the lock drops.
211func (s *Service) snapshotBindPolicies() map[string]domain.AppBindPolicy {
212 s.bindMu.RLock()
213 defer s.bindMu.RUnlock()
214 return s.bindPolicies
215}
216
217// WithDevicePolicies supplies the initial administrative device policies.
218func (s *Service) WithDevicePolicies(policies map[string]domain.AppDevicePolicy) *Service {
219 s.SetDevicePolicies(policies)
220 return s
221}
222
223// SetDevicePolicies atomically replaces the administrative device policies
224// used to authorize service devices. The map and its slices are copied so
225// later caller mutation cannot race an in-flight deploy.
226func (s *Service) SetDevicePolicies(policies map[string]domain.AppDevicePolicy) {
227 copied := make(map[string]domain.AppDevicePolicy, len(policies))
228 for name, policy := range policies {
229 policy.CDI = append([]string(nil), policy.CDI...)
230 policy.AllowedApps = append([]string(nil), policy.AllowedApps...)
231 policy.AllowedServices = append([]string(nil), policy.AllowedServices...)
232 copied[name] = policy
233 }
234 s.deviceMu.Lock()
235 s.devicePolicies = copied
236 s.deviceMu.Unlock()
237}
238
239// snapshotDevicePolicies returns the current policy map. SetDevicePolicies
240// never mutates a published map, so the snapshot stays safe after the
241// lock drops.
242func (s *Service) snapshotDevicePolicies() map[string]domain.AppDevicePolicy {
243 s.deviceMu.RLock()
244 defer s.deviceMu.RUnlock()
245 return s.devicePolicies
246}
247
248// resolveServiceDevices resolves every declared logical device against the
249// current administrative policies into runtime CDI IDs. Errors name only
250// the app, service, and device: host inventory never appears.
251func (s *Service) resolveServiceDevices(app string, svc domain.AppService) ([]string, error) {
252 if len(svc.Devices) == 0 {
253 return nil, nil
254 }
255 ids, err := domain.ResolveAppDevices(app, svc.Name, svc.Devices, s.snapshotDevicePolicies())
256 if err != nil {
257 return nil, fmt.Errorf("deployment: %w", err)
258 }
259 return ids, nil
260}
261
262// resolveServiceBinds resolves every declared bind against the current
263// administrative policies into ephemeral runtime binds. Errors name only the
264// app, service, and mount: host source paths never appear.
265func (s *Service) resolveServiceBinds(app string, svc domain.AppService) ([]domain.ContainerBind, error) {
266 if len(svc.Binds) == 0 {
267 return nil, nil
268 }
269 policies := s.snapshotBindPolicies()
270 resolved := make([]domain.ContainerBind, 0, len(svc.Binds))
271 for _, bind := range svc.Binds {
272 policy, ok := policies[bind.Name]
273 if !ok {
274 return nil, fmt.Errorf("deployment: app %q service %q mount %q: no administrative mount policy configured: %w", app, svc.Name, bind.Name, domain.ErrBindPolicy)
275 }
276 r, err := policy.ResolveAppBind(app, svc.Name, bind)
277 if err != nil {
278 return nil, fmt.Errorf("deployment: app %q service %q mount %q: refused by administrative mount policy: %w", app, svc.Name, bind.Name, domain.ErrBindPolicy)
279 }
280 resolved = append(resolved, domain.ContainerBind(r))
281 }
282 // Apply the runtime guard here so a bind overlapping a protected
283 // Gordon path fails at preflight and before the serving generation is
284 // retired, not only at create time. createContainer keeps the full
285 // guard as the final check.
286 if err := s.deps.Guard.Check(&domain.ContainerConfig{Binds: resolved}); err != nil {
287 return nil, fmt.Errorf("deployment: app %q service %q: %w", app, svc.Name, err)
288 }
289 return resolved, nil
290}
291
292// acquireAppContext takes the GC shared lease first and the per-app
293// coordinator second, following the documented lock order
294// (GC barrier → per-app coordinator → registry lock → bbolt
295// transaction). The returned release drops both, coordinator first.
296func (s *Service) acquireAppContext(ctx context.Context, app string) (func(), error) {
297 lease, err := s.acquireSharedGC(ctx)
298 if err != nil {
299 return nil, err
300 }
301 release, err := s.coord.acquire(ctx, app)
302 if err != nil {
303 lease.Release()
304 return nil, err
305 }
306 return func() {
307 release()
308 lease.Release()
309 }, nil
310}
311
312// tryAcquireAppContext takes the GC shared lease, then tries the per-app
313// coordinator without blocking. ok is false when the app is busy; the
314// caller skips it. Periodic reconciliation still waits for an in-flight
315// prune, because a half-planned snapshot is exactly what must not be
316// raced.
317func (s *Service) tryAcquireAppContext(ctx context.Context, app string) (func(), bool) {
318 lease, err := s.acquireSharedGC(ctx)
319 if err != nil {
320 return nil, false
321 }
322 release, ok := s.coord.tryAcquire(app)
323 if !ok {
324 lease.Release()
325 return nil, false
326 }
327 return func() {
328 release()
329 lease.Release()
330 }, true
331}
332
333// acquireSharedGC takes a shared GC lease, or a no-op lease when no
334// barrier is wired.
335func (s *Service) acquireSharedGC(ctx context.Context) (out.GCLease, error) {
336 if s.barrier == nil {
337 return noopGCLease{}, nil
338 }
339 return s.barrier.AcquireShared(ctx)
340}
341
342// noopGCLease is the lease used when no barrier is wired.
343type noopGCLease struct{}
344
345func (noopGCLease) Release() {}
346
347// WithProbeDeps overrides readiness probing (tests inject a mock-backed
348// ProbeDeps via NewProbeDeps; production leaves it nil for the live
349// runtime adapter).
350func (s *Service) WithProbeDeps(probes ProbeDeps) *Service {
351 s.probes = &probes
352 return s
353}
354
355// DeployInput selects the revision and service scope.
356type DeployInput struct {
357 App string
358 Revision string
359 Service string
360 Op string
361}
362
363// DeployResult carries terminal per-service results and cleanup warnings.
364type DeployResult struct {
365 Op string
366 App string
367 Revision string
368 Outcome string
369 Services map[string]ServiceResult
370 CleanupWarnings []CleanupWarning
371 // Removed names services retired because the new revision no longer
372 // declares them.
373 Removed []string
374}
375
376// ServiceResultUnchanged marks a deploy step that kept the running
377// container because its image, spec, and environment were already current.
378const ServiceResultUnchanged = domain.AppServiceUnchanged
379
380// ServiceResult is one service's terminal deployment result.
381type ServiceResult struct {
382 Result string
383 EffectiveRevision string
384 Before string
385 After string
386 RestartUnsafe bool
387 Error string
388 // VerifiedStartedAt is the execution start of After observed before its
389 // readiness ran. The caller records it as a verified execution once the
390 // generation is published and routed. Zero when unknown.
391 VerifiedStartedAt time.Time
392 // SupersededRunning is set by an overlap replacement: the container in
393 // Before still runs beside After and carries a retirement-pending marker
394 // until it is retired.
395 SupersededRunning bool
396 // BackendBinds carries the created container's TCP loopback publishes
397 // (container port -> 127.0.0.1 host port) for the active record.
398 BackendBinds map[int]int
399 // UDPBackendBinds carries the created container's UDP loopback
400 // publishes for the active record. Nil when the service declares
401 // no UDP interface.
402 UDPBackendBinds map[int]int
403 // CleanupWarnings are bounded leftovers of this service's terminal
404 // path: a candidate that could not be removed, or a backend claim
405 // that could not be released.
406 CleanupWarnings []CleanupWarning
407 // Diagnostics is bounded, redacted failure output of the failed
408 // candidate. It is never embedded in Error and is persisted to the
409 // journal for logs-scoped retrieval only.
410 Diagnostics []string
411}
412
413// CleanupWarning records post-publication leftovers for operator action.
414type CleanupWarning struct {
415 Service string
416 Leftover string
417 Detail string
418}
419
420// pinnedService binds a service spec to its resolved digest plus the
421// captured revision's app-wide public env (injected into every service
422// alongside that service's own resolved secrets). appEnv is a copy owned
423// by this pin: never mutated, never persisted with secret values.
424type pinnedService struct {
425 name string
426 spec domain.AppService
427 digest string
428 runtimeImage string
429 appEnv map[string]string
430 appNetworks []domain.AppSharedNetwork
431 // sharedNetworks are the declared shared-network memberships this
432 // service joins. The private incarnation network is derived from the
433 // app UUID at create time so recovery and deploy agree.
434 sharedNetworks []domain.AppSharedNetwork
435}
436
437// newOpID allocates a time-ordered op- identifier (uuid v7).
438func newOpID() string {
439 id, err := uuid.NewV7()
440 if err != nil {
441 id = uuid.New()
442 }
443 var buf [32]byte
444 hex.Encode(buf[:], id[:])
445 return "op-" + string(buf[:])
446}
447
448// claimDeploymentLocked is the first phase of a deploy: it resolves the
449// requested revision, enforces the targeted-deploy convergence rules, and
450// claims the request key, persisting the non-terminal journal. It performs
451// no image pull and no workload mutation, so a crash after it leaves a
452// durable claim that reconciliation can converge. owned is true only for the
453// single claimer of the key; a replay returns the stored journal instead.
454// The caller holds the app lock and has already recovered the store.
455func (s *Service) claimDeploymentLocked(ctx context.Context, input DeployInput) (domain.AppDesiredRevision, domain.AppOperation, bool, error) {
456 rev, err := s.resolveRevision(ctx, input)
457 if err != nil {
458 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
459 }
460 if input.Service != "" {
461 if err := s.checkConverged(ctx, input.App, input.Service, rev); err != nil {
462 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
463 }
464 }
465
466 op := domain.AppOperation{
467 Kind: "deploy",
468 App: input.App,
469 InputRevision: rev.Revision,
470 StartedAt: time.Now().UTC(),
471 Request: domain.AppOperationRequestFor("deploy", input.App, input.Revision, input.Service),
472 }
473 op, owned, err := s.claimOperation(ctx, input.Op, op, []domain.AppOperationStep{{ID: "preflight", State: domain.AppStepPending}})
474 if err != nil {
475 return domain.AppDesiredRevision{}, domain.AppOperation{}, false, err
476 }
477 return rev, op, owned, nil
478}
479
480// pinPreflightLocked is the second preflight phase: it runs the image
481// pull/pin and resource gates of an already claimed operation and records the
482// pinned table (or the terminal preflight failure) in the journal. The
483// caller holds the app lock and owns the claimed, non-terminal operation.
484func (s *Service) pinPreflightLocked(ctx context.Context, app, onlyService string, rev domain.AppDesiredRevision, op *domain.AppOperation) ([]pinnedService, error) {
485 log := zerowrap.FromCtx(ctx)
486
487 pinned, err := s.preflightServices(ctx, app, rev, onlyService)
488 if err != nil {
489 op.Steps[0] = domain.AppOperationStep{ID: "preflight", State: domain.AppStepFailed, Error: err.Error()}
490 op.Outcome = domain.AppOutcomeFailed
491 if saveErr := s.deps.State.SaveOperation(ctx, *op); saveErr != nil {
492 log.Warn().Err(saveErr).Msg("deployment: failed to record preflight failure")
493 }
494 return nil, err
495 }
496 op.Steps[0] = domain.AppOperationStep{ID: "preflight", State: domain.AppStepSucceeded}
497 // A resumed claim can carry the service-step plan of an earlier attempt,
498 // including candidates it recorded. Converge those candidates before
499 // replacing the plan: an unreconciled leftover may still be running, and
500 // the replacement this execution creates must never overlap it.
501 if err := s.convergeResumedCandidates(ctx, app, op, log); err != nil {
502 return nil, err
503 }
504 // Replace the persisted plan instead of appending: the preflight step plus
505 // one step per pinned service, so runServiceStep's index contract matches a
506 // normalized layout and no earlier step is overwritten or left pending.
507 plan := make([]domain.AppOperationStep, 1, len(pinned)+1)
508 plan[0] = op.Steps[0]
509 for _, p := range pinned {
510 plan = append(plan, domain.AppOperationStep{
511 ID: "service." + p.name + ".replace",
512 State: domain.AppStepPending,
513 Service: p.name,
514 Digest: p.digest,
515 Image: p.runtimeImage,
516 })
517 }
518 op.Steps = plan
519 if err := s.deps.State.SaveOperation(ctx, *op); err != nil {
520 return nil, fmt.Errorf("deployment: persist pinned table: %w", err)
521 }
522 log.Info().Str("op", op.Op).Str("revision", rev.Revision).Int("services", len(pinned)).Msg("deployment: preflight passed")
523 return pinned, nil
524}
525
526// convergeResumedCandidates converges any candidate an earlier attempt of a
527// resumed claim recorded, so replacing its service-step plan cannot drop a
528// leftover that may still be running. A fresh claim has only its preflight
529// step and does no work here.
530func (s *Service) convergeResumedCandidates(ctx context.Context, app string, op *domain.AppOperation, log zerowrap.Logger) error {
531 needsActive := false
532 for _, step := range op.Steps {
533 if reconcileNeedsContainer(step, false) {
534 needsActive = true
535 break
536 }
537 }
538 if !needsActive {
539 return nil
540 }
541 active, _, err := s.deps.State.LoadActive(ctx, app)
542 if err != nil {
543 return fmt.Errorf("deployment: load active for resumed claim: %w", err)
544 }
545 if _, err := s.convergeInterruptedSteps(ctx, app, op, active, false, log); err != nil {
546 return err
547 }
548 return nil
549}
550
551// claimOperation is the single claim point before any mutation effect.
552// A keyed request is claimed atomically: the first claim persists the
553// in-flight journal and reports owned=true; a repeat returns the stored
554// journal and reports owned=false. A key that already answered a
555// different request fails with ErrAppStateConflict, and a key for a name
556// with no app identity fails with ErrAppNotFound without writing
557// anything. An unkeyed request (internal recovery paths) starts a fresh
558// journal under a generated id.
559func (s *Service) claimOperation(ctx context.Context, key string, op domain.AppOperation, steps []domain.AppOperationStep) (domain.AppOperation, bool, error) {
560 op.Steps = steps
561 // A journal is never opened while a different operation of this app is
562 // still unfinished: that predecessor must be reconciled first, or this
563 // claim would mask it and its leftover generation could stay untracked.
564 latest, found, err := s.deps.State.LoadLatestOperation(ctx, op.App)
565 if err != nil {
566 return domain.AppOperation{}, false, fmt.Errorf("deployment: load latest operation: %w", err)
567 }
568 if found && !latest.Terminal() && latest.Op != key {
569 return domain.AppOperation{}, false, fmt.Errorf(
570 "deployment: app %q has unfinished operation %s: %w", op.App, latest.Op, domain.ErrAppStateConflict)
571 }
572 if key == "" {
573 op.Op = newOpID()
574 if err := s.deps.State.SaveOperation(ctx, op); err != nil {
575 return domain.AppOperation{}, false, fmt.Errorf("deployment: persist journal before effects: %w", err)
576 }
577 return op, true, nil
578 }
579 op.Op = key
580 existing, claimed, err := s.deps.State.ClaimOperation(ctx, op)
581 if err != nil {
582 return domain.AppOperation{}, false, err
583 }
584 if !claimed {
585 return existing, false, nil
586 }
587 return op, true, nil
588}
589
590// replayError reports a replayed key whose journal never reached a
591// terminal outcome: the operation is still in flight or was interrupted,
592// so its effects must not run again.
593func replayError(op domain.AppOperation) error {
594 if !op.Terminal() {
595 return fmt.Errorf("deployment: operation %s has not reached a terminal outcome: %w", op.Op, domain.ErrAppStateConflict)
596 }
597 if op.Outcome != domain.AppOutcomeSuccess {
598 return fmt.Errorf("deployment: operation %s previously ended with outcome %s: %w", op.Op, op.Outcome, domain.ErrAppStateConflict)
599 }
600 return nil
601}
602
603// resolveRevision loads the captured revision (default: current desired).
604// A revision that cannot be resolved on a name with no app identity at
605// all reports ErrAppNotFound: the failure is the missing app, not the
606// missing revision.
607func (s *Service) resolveRevision(ctx context.Context, input DeployInput) (domain.AppDesiredRevision, error) {
608 if input.Revision != "" {
609 rev, err := s.deps.State.LoadRevision(ctx, input.App, input.Revision)
610 if err != nil && errors.Is(err, domain.ErrAppRevisionNotFound) {
611 if knownErr := s.requireKnownApp(ctx, input.App); knownErr != nil {
612 return domain.AppDesiredRevision{}, knownErr
613 }
614 }
615 return rev, err
616 }
617 rev, ok, err := s.deps.State.LoadDesired(ctx, input.App)
618 if err != nil {
619 return domain.AppDesiredRevision{}, err
620 }
621 if !ok {
622 if knownErr := s.requireKnownApp(ctx, input.App); knownErr != nil {
623 return domain.AppDesiredRevision{}, knownErr
624 }
625 return domain.AppDesiredRevision{}, fmt.Errorf("deployment: app %q has no desired state: %w", input.App, domain.ErrAppRevisionNotFound)
626 }
627 return rev, nil
628}
629
630// requireKnownApp refuses a mutation of a name with no live app identity.
631func (s *Service) requireKnownApp(ctx context.Context, app string) error {
632 exists, err := s.deps.State.AppExists(ctx, app)
633 if err != nil {
634 return fmt.Errorf("deployment: app existence: %w", err)
635 }
636 if !exists {
637 return fmt.Errorf("deployment: app %q does not exist: %w", app, domain.ErrAppNotFound)
638 }
639 return nil
640}
641
642// checkConverged permits a targeted deploy when every desired/effective
643// difference belongs to that service. App-wide and other-service changes
644// must be deployed together so ACTIVE never combines incompatible specs.
645func (s *Service) checkConverged(ctx context.Context, app, service string, rev domain.AppDesiredRevision) error {
646 active, ok, err := s.deps.State.LoadActive(ctx, app)
647 if err != nil {
648 return err
649 }
650 if !ok {
651 return fmt.Errorf("deployment: app %q was never deployed, targeted deploy refused: %w", app, domain.ErrAppStateConflict)
652 }
653 if active.Converged && active.ConvergedRevision == rev.Revision {
654 return nil
655 }
656 effective, err := domain.ActiveAppSpec(active, rev.Spec.Env, func(revision string) (map[string]string, error) {
657 return s.revisionEnv(ctx, app, revision)
658 })
659 if err != nil {
660 return fmt.Errorf("deployment: rebuild effective spec: %w", err)
661 }
662 diff := domain.DiffAppSpec(rev.Spec, effective)
663 prefix := "service/" + service + "/"
664 if len(diff.Added) > 0 {
665 return targetedDivergenceError(rev.Revision, diff.Added[0]+" (added)")
666 }
667 if len(diff.Removed) > 0 {
668 return targetedDivergenceError(rev.Revision, diff.Removed[0]+" (removed)")
669 }
670 for _, path := range diff.Changed {
671 if !strings.HasPrefix(path, prefix) {
672 return targetedDivergenceError(rev.Revision, path)
673 }
674 }
675 return nil
676}
677
678// revisionEnv returns the app env of a stored revision, or nil when the
679// revision was garbage-collected.
680func (s *Service) revisionEnv(ctx context.Context, app, revision string) (map[string]string, error) {
681 rev, err := s.deps.State.LoadRevision(ctx, app, revision)
682 if errors.Is(err, domain.ErrAppRevisionNotFound) {
683 return nil, nil
684 }
685 if err != nil {
686 return nil, err
687 }
688 return rev.Spec.Env, nil
689}
690
691func targetedDivergenceError(revision, path string) error {
692 return fmt.Errorf("%w: service-targeted deploy refused, desired %s also changes %s", domain.ErrAppStateConflict, revision, path)
693}
694
695// selectPreflightServices copies and optionally narrows a revision's services.
696func selectPreflightServices(rev domain.AppDesiredRevision, onlyService string) ([]domain.AppService, error) {
697 services := append([]domain.AppService(nil), rev.Spec.Services...)
698 if onlyService == "" {
699 return services, nil
700 }
701 for _, svc := range services {
702 if svc.Name == onlyService {
703 return []domain.AppService{svc}, nil
704 }
705 }
706 return nil, fmt.Errorf("deployment: service %q not in revision %s: %w", onlyService, rev.Revision, domain.ErrAppStateConflict)
707}
708
709// preflightServices runs the preflight gates in order. No mutation.
710func (s *Service) preflightServices(ctx context.Context, app string, rev domain.AppDesiredRevision, onlyService string) ([]pinnedService, error) {
711 services, err := selectPreflightServices(rev, onlyService)
712 if err != nil {
713 return nil, err
714 }
715 sort.Slice(services, func(i, j int) bool { return services[i].Name < services[j].Name })
716
717 var pinned []pinnedService
718 ownership, err := s.deps.State.LoadOwnership(ctx, app)
719 if err != nil {
720 return nil, err
721 }
722 // Defensive re-check: stored revisions predate or bypass apply
723 // validation; an overlap must fail preflight before any mutation.
724 if err := rev.Spec.CheckEnvSecretCollisions(); err != nil {
725 return nil, err
726 }
727 appEnv := maps.Clone(rev.Spec.Env)
728 if appEnv == nil {
729 appEnv = map[string]string{}
730 }
731 for _, svc := range services {
732 digest, err := s.deps.Images.ResolveDigest(ctx, svc.Image)
733 if err != nil {
734 return nil, fmt.Errorf("deployment: image %q unresolvable: %w", svc.Image, domain.ErrAppImageUnresolvable)
735 }
736 if err := s.checkSecrets(ctx, app, ownership.ID, svc); err != nil {
737 return nil, err
738 }
739 runtimeImage, err := s.preflightImage(ctx, svc.Image, digest)
740 if err != nil {
741 return nil, err
742 }
743 if err := s.checkImageVolumes(ctx, svc, runtimeImage); err != nil {
744 return nil, err
745 }
746 // Bind resolution stays a preflight gate: an unresolvable bind
747 // must fail before any workload mutation. The successful values
748 // are discarded because createAndStart re-resolves against the
749 // current policy immediately before the runtime mutation.
750 if _, err := s.resolveServiceBinds(app, svc); err != nil {
751 return nil, err
752 }
753 if err := s.preflightServiceDevices(ctx, app, svc, pinned); err != nil {
754 return nil, err
755 }
756 pinned = append(pinned, pinnedService{
757 name: svc.Name, spec: svc, digest: digest, runtimeImage: runtimeImage, appEnv: appEnv,
758 appNetworks: append([]domain.AppSharedNetwork(nil), rev.Spec.Networks...),
759 sharedNetworks: domain.AppServiceSharedNetworks(rev.Spec, svc.Name),
760 })
761 }
762 if err := s.recheckReservations(ctx, app, rev); err != nil {
763 return nil, err
764 }
765 if err := s.checkResources(ctx, app, ownership, services); err != nil {
766 return nil, err
767 }
768 return pinned, nil
769}
770
771// preflightServiceDevices runs the device authorization and engine
772// capability gates for one service. Split from preflightServices to keep
773// its complexity within budget.
774func (s *Service) preflightServiceDevices(ctx context.Context, app string, svc domain.AppService, pinned []pinnedService) error {
775 if _, err := s.resolveServiceDevices(app, svc); err != nil {
776 return err
777 }
778 // Engine capability is a knowable-before-mutation failure: a
779 // device-bearing revision on an unsupported engine must fail here,
780 // before withdrawal retires the serving generation. The check runs
781 // once per revision, on the first device-bearing service;
782 // CreateContainer keeps the same gate as defense.
783 if len(svc.Devices) > 0 && !devicesEngineChecked(pinned) {
784 if s.deps.Runtime == nil {
785 return fmt.Errorf("deployment: runtime unavailable: %w", domain.ErrRuntimeUnsupported)
786 }
787 // Sanitize the adapter cause: version-probe failures may embed
788 // the daemon endpoint, which must not reach journals or API
789 // responses. Cancellation and deadlines are the exception: they
790 // stay recognizable so aborted deploys are not misread as
791 // incapable engines.
792 if err := s.deps.Runtime.SupportsCDIDevices(ctx); err != nil {
793 return sanitizedEngineProbeError(svc.Name, err)
794 }
795 }
796 return nil
797}
798
799// sanitizedEngineProbeError maps an engine device-capability probe failure
800// onto the caller-visible error. Cancellation and deadlines survive so
801// shutdown and timeout handling keep working; every other cause collapses
802// to ErrRuntimeUnsupported alone, dropping the adapter text that may embed
803// the daemon endpoint.
804func sanitizedEngineProbeError(service string, err error) error {
805 switch {
806 case errors.Is(err, context.Canceled):
807 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, context.Canceled)
808 case errors.Is(err, context.DeadlineExceeded):
809 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, context.DeadlineExceeded)
810 default:
811 return fmt.Errorf("deployment: engine device capability check for service %q: %w", service, domain.ErrRuntimeUnsupported)
812 }
813}
814
815// devicesEngineChecked reports whether an earlier pinned service already
816// triggered the engine capability probe for this revision.
817func devicesEngineChecked(pinned []pinnedService) bool {
818 for _, p := range pinned {
819 if len(p.spec.Devices) > 0 {
820 return true
821 }
822 }
823 return false
824}
825
826// checkSecrets reads every required secret path once. Values stay in
827// memory for container creation only; nothing is written anywhere.
828// Paths are UUID-keyed so a removed app's secrets are never adopted
829// by a new app reusing the name.
830func (s *Service) checkSecrets(ctx context.Context, app, appID string, svc domain.AppService) error {
831 if len(svc.SecretFiles) > 0 && s.deps.SecretFiles == nil {
832 return fmt.Errorf("deployment: service %q delivers secrets as files but no secret file store is configured", svc.Name)
833 }
834 if s.deps.Secrets == nil {
835 return fmt.Errorf("deployment: secret provider unavailable: %w", domain.ErrAppSecretMissing)
836 }
837 for envKey, name := range svc.Secrets {
838 path := domain.AppSecretPathForID(appID, app, svc.Name, name)
839 if _, err := s.deps.Secrets.GetSecret(ctx, path); err != nil {
840 return fmt.Errorf("deployment: secret %s (env %s) missing: %w", path, envKey, domain.ErrAppSecretMissing)
841 }
842 }
843 return nil
844}
845
846// appSecretID returns the stable internal UUID for secret paths.
847// Empty means legacy name-keyed paths (records predating UUIDs).
848// Used on the execution path where preflight's ownership snapshot is
849// not in scope; deploys are infrequent, one extra store read per
850// service is acceptable.
851func (s *Service) appSecretID(ctx context.Context, app string) (string, error) {
852 ownership, err := s.deps.State.LoadOwnership(ctx, app)
853 if err != nil {
854 return "", err
855 }
856 return ownership.ID, nil
857}
858
859func (s *Service) preflightImage(ctx context.Context, image, digest string) (string, error) {
860 if err := validation.ValidateImageDigest(digest); err != nil {
861 return "", fmt.Errorf("deployment: image %q resolved to invalid digest: %w", image, domain.ErrAppImageUnresolvable)
862 }
863 if err := s.validateImageSource(image, digest); err != nil {
864 return "", err
865 }
866 if s.deps.Registry.Domain == "" {
867 return image, nil
868 }
869 return s.pullImage(ctx, stripImageTag(image)+"@"+digest)
870}
871
872// validateImageSource enforces the installation image policy on a
873// reference before any pull. Both the manifest reference and the pinned
874// digest are checked, so a digest-pinned ref cannot bypass the hostname
875// and port allowlist on deploy or recovery.
876func (s *Service) validateImageSource(image, digest string) error {
877 ref := strings.TrimSpace(image)
878 if digest != "" && !strings.Contains(ref, "@") {
879 ref = stripImageTag(image) + "@" + digest
880 }
881 if err := s.deps.ImagePolicy.ValidateImageSource(ref); err != nil {
882 return fmt.Errorf("deployment: image %q: %w", image, err)
883 }
884 return nil
885}
886
887// checkImageVolumes rejects images with unmapped VOLUME declarations.
888func (s *Service) checkImageVolumes(ctx context.Context, svc domain.AppService, runtimeImage string) error {
889 declared, err := s.deps.Runtime.InspectImageVolumes(ctx, runtimeImage)
890 if err != nil {
891 return fmt.Errorf("deployment: inspect image volumes: %w", err)
892 }
893 if len(declared) == 0 {
894 return nil
895 }
896 mapped := map[string]struct{}{}
897 for _, vol := range svc.Volumes {
898 mapped[vol.Path] = struct{}{}
899 }
900 // Bind destinations also map image-declared volumes: a bind that
901 // mounts over an image VOLUME is an explicit operator mapping.
902 for _, bind := range svc.Binds {
903 mapped[bind.Path] = struct{}{}
904 }
905 for _, path := range declared {
906 if _, ok := mapped[path]; !ok {
907 return fmt.Errorf(
908 "deployment: image declares unmanaged volume %q, map it explicitly: %w",
909 path, domain.ErrAppUnmanagedImageVolume,
910 )
911 }
912 }
913 return nil
914}
915
916// recheckReservations re-validates against the live global table.
917func (s *Service) recheckReservations(ctx context.Context, app string, rev domain.AppDesiredRevision) error {
918 checkpoint, err := s.deps.State.LoadCheckpoint(ctx)
919 if err != nil {
920 return err
921 }
922 reservations := domain.ReservationsFor(rev.Spec)
923 if err := domain.CheckReservations(checkpoint.Reservations, reservations, app); err != nil {
924 return fmt.Errorf("deployment: %w", err)
925 }
926 return nil
927}
928
929// checkResources verifies volumes exist-or-creatable and ownership is consistent.
930// A generated runtime volume that already exists but is NOT recorded in
931// this app's ownership is refused fail-closed: mounting it would
932// implicitly adopt (and possibly modify) foreign data. Volumes recorded
933// in ownership (attached or retained) were created by this app's own
934// deploys and are safe to reuse.
935func (s *Service) checkResources(ctx context.Context, app string, ownership domain.AppOwnership, services []domain.AppService) error {
936 owned := map[string]string{}
937 ownedRuntime := map[string]struct{}{}
938 for _, vol := range ownership.Volumes {
939 owned[vol.Name] = vol.Service
940 ownedRuntime[vol.RuntimeName] = struct{}{}
941 }
942 for _, svc := range services {
943 for _, vol := range svc.Volumes {
944 if owner, ok := owned[vol.Name]; ok && owner != svc.Name {
945 return fmt.Errorf(
946 "deployment: volume %q owned by %q, not %q: %w",
947 vol.Name, owner, svc.Name, domain.ErrAppStateConflict,
948 )
949 }
950 runtimeName := domain.RuntimeVolumeName(app, svc.Name, vol.Name)
951 exists, err := s.deps.Runtime.VolumeExists(ctx, runtimeName)
952 if err != nil {
953 return fmt.Errorf("deployment: check volume %q: %w", runtimeName, err)
954 }
955 if exists {
956 if _, ok := ownedRuntime[runtimeName]; !ok {
957 return fmt.Errorf(
958 "deployment: runtime volume %q already exists but is not owned by app %q: refusing to adopt foreign data: %w",
959 runtimeName, app, domain.ErrAppStateConflict,
960 )
961 }
962 }
963 }
964 }
965 return nil
966}
967
968// ComputeOutcome derives the op outcome over terminal per-service results.
969func ComputeOutcome(results map[string]ServiceResult) string {
970 deployed := 0
971 failed := 0
972 for _, result := range results {
973 switch result.Result {
974 case "deployed", ServiceResultUnchanged, domain.AppServiceRemoved:
975 deployed++
976 case "failed":
977 failed++
978 }
979 }
980 switch {
981 case deployed > 0 && failed > 0:
982 return domain.AppOutcomePartial
983 case deployed > 0:
984 return domain.AppOutcomeSuccess
985 default:
986 return domain.AppOutcomeFailed
987 }
988}
989
990// itoa formats integers without extra imports.
991func itoa(n int) string {
992 return fmt.Sprintf("%d", n)
993}