1package deployment
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "maps"
8 "slices"
9 "sort"
10 "strings"
11 "time"
12
13 "github.com/bnema/zerowrap"
14
15 "github.com/bnema/gordon/internal/domain"
16)
17
18// startedCandidate is one freshly started replacement generation and its
19// observed loopback backend binds (container port -> host port).
20type startedCandidate struct {
21 Container *domain.Container
22 TCPBinds map[int]int
23 UDPBinds map[int]int
24 // StartedAt is the execution start observed right after the container
25 // was started, before readiness ran. Zero when it could not be read.
26 StartedAt time.Time
27}
28
29// createAndStart builds the runtime config and starts the replacement.
30// Preflight already pulled and inspected the pinned runtime image.
31// Every TCP-capable interface port is published on 127.0.0.1 ephemeral
32// plus every UDP interface port on 127.0.0.1/udp ephemeral: readiness
33// and the proxy dial these loopback binds rootless-first,
34// never container IPs. The runtime keeps native restarts; Gordon
35// reconciles intent at boot and in the monitor (accepted decision:
36// native runtime restarts plus daemon reconciliation).
37func (s *Service) createAndStart(ctx context.Context, app, revision string, p pinnedService, opID string, journal candidateJournal) (startedCandidate, error) {
38 // Re-resolve binds from the current policy immediately before any
39 // runtime mutation: a bind revoked since preflight must fail here,
40 // before volume ownership or container creation.
41 resolvedBinds, err := s.resolveServiceBinds(app, p.spec)
42 if err != nil {
43 return startedCandidate{}, err
44 }
45 // Re-resolve devices and capabilities from the current policy under
46 // the same fail-closed rule: revoked grants never reach the runtime.
47 resolvedDevices, resolvedCaps, err := s.resolveServiceGrants(app, p.spec)
48 if err != nil {
49 return startedCandidate{}, err
50 }
51 env, files, err := s.serviceEnv(ctx, app, p)
52 if err != nil {
53 return startedCandidate{}, err
54 }
55 image := runtimeImageRef(p)
56 // The incarnation network is verified or created before any
57 // container exists, so a workload is only ever started on a
58 // Gordon-owned network that no other app shares.
59 appID, err := s.ensureIncarnationID(ctx, app)
60 if err != nil {
61 return startedCandidate{}, err
62 }
63 nets := s.resolveAppNetworks(appID, p.sharedNetworks)
64 if err := s.ensureAppNetworks(ctx, app, appID, nets); err != nil {
65 return startedCandidate{}, err
66 }
67 volumes := map[string]string{}
68 readOnlyVolumes := map[string]string{}
69 for _, vol := range p.spec.Volumes {
70 runtimeName := domain.RuntimeVolumeName(app, p.spec.Name, vol.Name)
71 // Durable ownership is reserved BEFORE the runtime volume
72 // exists: a crash in between leaves a protected record, never
73 // an unowned volume that prune could later adopt. The
74 // incarnation ID is already validated above, so the labels
75 // carry the same ID without another ownership load.
76 if err := s.reserveVolumeOwnership(ctx, app, appID, p.spec.Name, vol.Name, runtimeName); err != nil {
77 return startedCandidate{}, err
78 }
79 if err := s.deps.Runtime.CreateVolume(ctx, runtimeName, volumeProvenanceLabels(app, appID, p.spec.Name, revision)); err != nil {
80 // CreateVolume is idempotent at the adapter; existence was
81 // checked at preflight, so only real backend errors fail here.
82 return startedCandidate{}, fmt.Errorf("deployment: create volume %q: %w", vol.Name, err)
83 }
84 // A declared read-only mount must reach the runtime as read-only:
85 // a writable mount would let the service modify protected data.
86 if vol.ReadOnly {
87 readOnlyVolumes[vol.Path] = runtimeName
88 continue
89 }
90 volumes[vol.Path] = runtimeName
91 }
92 config := &domain.ContainerConfig{
93 Image: image,
94 Name: domain.LogicalServiceIdentity(app, p.spec.Name) + "--" + shortOp(opID),
95 Env: env,
96 Entrypoint: append([]string(nil), p.spec.Command...),
97 Volumes: volumes,
98 ReadOnlyVolumes: readOnlyVolumes,
99 Binds: resolvedBinds,
100 CDIDevices: resolvedDevices,
101 AddedCapabilities: resolvedCaps,
102 Labels: appLabels(app, p.spec.Name, revision),
103 AutoRemove: false,
104 RestartPolicy: domain.RestartPolicyAlways,
105 PortPublishes: backendPublishes(p.spec),
106 NetworkMode: nets.private,
107 Hostname: p.spec.Name,
108 Aliases: []string{p.spec.Name},
109 MemoryLimit: s.deps.Limits.MemoryBytes,
110 NanoCPUs: s.deps.Limits.NanoCPUs,
111 PidsLimit: s.deps.Limits.PidsLimit,
112 }
113 created, err := s.createWithSecretFiles(ctx, app, p.name, config, files)
114 if err != nil {
115 return startedCandidate{}, err
116 }
117 // The candidate ID is durably journaled before the container is started
118 // or joins a shared network, so an interrupted deployment can be cleaned
119 // up instead of leaving an untracked running generation.
120 if err := s.recordCandidate(ctx, app, p.spec.Name, created.ID, journal); err != nil {
121 return startedCandidate{}, err
122 }
123 if err := s.connectSharedNetworks(ctx, created.ID, nets); err != nil {
124 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
125 return startedCandidate{Container: created}, err
126 }
127 if err := s.deps.Runtime.StartContainer(ctx, created.ID); err != nil {
128 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
129 return startedCandidate{Container: created}, fmt.Errorf("deployment: start container: %w", err)
130 }
131 // The execution is observed before readiness runs: a native restart
132 // between readiness and the verified-execution record then shows up as a
133 // different start time, so it is still treated as a fresh execution.
134 startedAt := s.executionStart(ctx, created.ID)
135 binds, udpBinds, err := s.readBackendBinds(ctx, app, p.spec.Name, created.ID, backendPorts(p.spec))
136 if err != nil {
137 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
138 return startedCandidate{Container: created}, err
139 }
140 return startedCandidate{Container: created, TCPBinds: binds, UDPBinds: udpBinds, StartedAt: startedAt}, nil
141}
142
143// runtimeImageRef resolves the exact runtime image reference of one pinned
144// service: the image preflight resolved, else the manifest reference pinned to
145// its digest.
146func runtimeImageRef(p pinnedService) string {
147 if p.runtimeImage != "" {
148 return p.runtimeImage
149 }
150 if p.digest == "" {
151 return p.spec.Image
152 }
153 return stripImageTag(p.spec.Image) + "@" + p.digest
154}
155
156// recordCandidate durably journals a freshly created candidate before it is
157// started. A candidate whose ID cannot be recorded must not keep running:
158// recovery could never find it, so it is removed and the failure surfaced.
159func (s *Service) recordCandidate(ctx context.Context, app, service, containerID string, journal candidateJournal) error {
160 if journal == nil {
161 return nil
162 }
163 if err := journal(ctx, containerID); err != nil {
164 retired := s.retireContainer(ctx, app, retireOptions{Service: service, Force: true}, containerID)
165 if !retired.Gone {
166 return fmt.Errorf("%w (candidate cleanup: %s)", err, cleanupDetail(retired))
167 }
168 return err
169 }
170 return nil
171}
172
173// createWithSecretFiles writes file-delivered secrets to the host, mounts
174// each one read-only at /run/secrets/<KEY>, and creates the container.
175// The files are removed again when the create fails. Values stay in
176// memory and on the 0444 host files only; nothing is logged.
177func (s *Service) createWithSecretFiles(ctx context.Context, app, service string, config *domain.ContainerConfig, files map[string]string) (*domain.Container, error) {
178 if len(files) == 0 {
179 return s.createContainer(ctx, service, config)
180 }
181 if s.deps.SecretFiles == nil {
182 return nil, fmt.Errorf("deployment: service %q delivers secrets as files but no secret file store is configured", service)
183 }
184 paths, err := s.deps.SecretFiles.Write(ctx, app, config.Name, files)
185 if err != nil {
186 return nil, fmt.Errorf("deployment: write secret files for service %q: %w", service, err)
187 }
188 withFiles := *config
189 withFiles.Binds = append([]domain.ContainerBind(nil), config.Binds...)
190 for _, key := range slices.Sorted(maps.Keys(files)) {
191 withFiles.Binds = append(withFiles.Binds, domain.ContainerBind{
192 Name: "secret-file-" + key,
193 Source: paths[key],
194 Destination: domain.SecretFilePath(key),
195 ReadOnly: true,
196 })
197 }
198 created, err := s.createContainer(ctx, service, &withFiles)
199 if err != nil {
200 if rmErr := s.deps.SecretFiles.Remove(ctx, app, config.Name); rmErr != nil {
201 log := zerowrap.FromCtx(ctx)
202 log.Warn().Str("app", app).Str("service", service).
203 Msg("deployment: secret files cleanup after failed create failed")
204 }
205 return nil, err
206 }
207 return created, nil
208}
209
210func (s *Service) createContainer(ctx context.Context, service string, config *domain.ContainerConfig) (*domain.Container, error) {
211 if err := s.deps.Guard.Check(config); err != nil {
212 return nil, fmt.Errorf("deployment: create container for service %q: %w", service, err)
213 }
214 created, err := s.deps.Runtime.CreateContainer(ctx, config)
215 if err == nil {
216 return created, nil
217 }
218 // The engine-unsupported sentinel survives redaction so callers can
219 // map it to the structured runtime-unsupported envelope. It carries
220 // no host inventory (engine family/version only), and the device
221 // gate runs only for device-bearing creates, so this branch also
222 // covers services that declare both binds and devices.
223 if errors.Is(err, domain.ErrRuntimeUnsupported) {
224 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: %w", service, domain.ErrRuntimeUnsupported)
225 }
226 if len(config.Binds) > 0 {
227 // A CreateContainer failure is a runtime error, not a bind policy
228 // violation: policy was already enforced by resolveServiceBinds
229 // before this call. Runtime errors may embed the resolved host
230 // path, so redact the whole cause instead of mislabelling it and
231 // keep it out of operation journals and API/CLI responses.
232 return nil, fmt.Errorf("deployment: create container for service %q with administrative mounts: runtime error redacted", service)
233 }
234 if len(config.CDIDevices) > 0 {
235 // Same redaction rule for device-bearing creates: runtime errors
236 // may embed resolved CDI IDs, which are host inventory. Policy
237 // was already enforced by resolveServiceDevices before this call.
238 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: runtime error redacted", service)
239 }
240 return nil, fmt.Errorf("deployment: create container: %w", err)
241}
242
243// reserveVolumeOwnership durably records one app-owned volume as
244// attached BEFORE the runtime volume is created, so a crash between the
245// two leaves a protected record instead of an unowned volume. The
246// caller passes the validated incarnation ID from ensureIncarnationID,
247// which already assigned it, so no reload is needed for the labels.
248func (s *Service) reserveVolumeOwnership(ctx context.Context, app, appID, service, volumeName, runtimeName string) error {
249 ownership, err := s.deps.State.LoadOwnership(ctx, app)
250 if err != nil {
251 return fmt.Errorf("deployment: load ownership: %w", err)
252 }
253 if ownership.App == "" {
254 ownership.App = app
255 }
256 if ownership.ID == "" {
257 ownership.ID = appID
258 }
259 entry := domain.AppOwnedVolume{
260 Name: volumeName,
261 Service: service,
262 RuntimeName: runtimeName,
263 State: domain.AppResourceAttached,
264 }
265 replaced := false
266 for i, existing := range ownership.Volumes {
267 if existing.RuntimeName == runtimeName {
268 ownership.Volumes[i] = entry
269 replaced = true
270 break
271 }
272 }
273 if !replaced {
274 ownership.Volumes = append(ownership.Volumes, entry)
275 }
276 if err := s.deps.State.SaveOwnership(ctx, ownership); err != nil {
277 return fmt.Errorf("deployment: reserve volume ownership: %w", err)
278 }
279 return nil
280}
281
282// volumeProvenanceLabels is the label set stamped on an app-created
283// volume. Ownership is never inferred from the volume name, so the
284// labels must carry the full provenance: app, incarnation UUID, and
285// service. The managed marker is stamped here as well as merged by the
286// adapter, so provenance never depends on adapter behaviour alone.
287func volumeProvenanceLabels(app, appID, service, revision string) map[string]string {
288 labels := map[string]string{
289 domain.LabelManaged: "true",
290 domain.LabelApp: app,
291 domain.LabelAppService: service,
292 domain.LabelAppRevision: revision,
293 }
294 if appID != "" {
295 labels[domain.LabelAppID] = appID
296 }
297 return labels
298}
299
300// appLabels stamps engine ownership on every created container.
301func appLabels(app, service, revision string) map[string]string {
302 return map[string]string{
303 domain.LabelManaged: "true",
304 domain.LabelApp: app,
305 domain.LabelAppService: service,
306 domain.LabelAppRevision: revision,
307 }
308}
309
310// shortOp keeps container names short but unique per op.
311func shortOp(opID string) string {
312 trimmed := strings.TrimPrefix(opID, "op-")
313 if len(trimmed) > 12 {
314 trimmed = trimmed[:12]
315 }
316 if trimmed == "" {
317 return "op"
318 }
319 return trimmed
320}
321
322// serviceEnv resolves the service environment: the captured revision's
323// app-wide public env on every service, overlaid by that service's own
324// public env (the service value wins for the same key), plus its secret
325// values (memory only, UUID-keyed paths). Keys are emitted sorted and
326// deterministic. Overlapping public/secret keys are invalid with no
327// precedence; preflight rejects them before any mutation.
328//
329// File-delivered secrets are returned in files (env key -> value, nil
330// when none) and the env gets <KEY>_FILE=/run/secrets/<KEY> instead.
331func (s *Service) serviceEnv(ctx context.Context, app string, p pinnedService) ([]string, map[string]string, error) {
332 spec := p.spec
333 envMap := make(map[string]string, len(p.appEnv)+len(spec.Env)+len(spec.Secrets))
334 maps.Copy(envMap, p.appEnv)
335 maps.Copy(envMap, spec.Env)
336 keys := make([]string, 0, len(spec.Secrets))
337 for envKey := range spec.Secrets {
338 keys = append(keys, envKey)
339 }
340 sort.Strings(keys)
341 if len(keys) == 0 {
342 return sortedEnv(envMap), nil, nil
343 }
344 id, err := s.appSecretID(ctx, app)
345 if err != nil {
346 return nil, nil, err
347 }
348 var files map[string]string
349 for _, envKey := range keys {
350 if err := checkSecretEnvOverlap(envMap, spec, envKey); err != nil {
351 return nil, nil, err
352 }
353 path := domain.AppSecretPathForID(id, app, spec.Name, spec.Secrets[envKey])
354 value, err := s.deps.Secrets.GetSecret(ctx, path)
355 if err != nil {
356 return nil, nil, fmt.Errorf("deployment: secret %s (env %s) missing: %w", path, envKey, domain.ErrAppSecretMissing)
357 }
358 if !spec.SecretAsFile(envKey) {
359 envMap[envKey] = value
360 continue
361 }
362 if files == nil {
363 files = make(map[string]string, len(spec.SecretFiles))
364 }
365 files[envKey] = value
366 envMap[domain.SecretFileEnvKey(envKey)] = domain.SecretFilePath(envKey)
367 }
368 return sortedEnv(envMap), files, nil
369}
370
371// checkSecretEnvOverlap refuses a secret key (and, for file delivery, its
372// <KEY>_FILE key) that is already set by public env or another secret.
373func checkSecretEnvOverlap(envMap map[string]string, spec domain.AppService, envKey string) error {
374 if _, overlap := envMap[envKey]; overlap {
375 return fmt.Errorf("deployment: env key %q collides with a secret key in service %q: %w", envKey, spec.Name, domain.ErrInvalidAppSpec)
376 }
377 if !spec.SecretAsFile(envKey) {
378 return nil
379 }
380 fileKey := domain.SecretFileEnvKey(envKey)
381 if _, overlap := envMap[fileKey]; overlap {
382 return fmt.Errorf("deployment: env key %q collides with a secret file key in service %q: %w", fileKey, spec.Name, domain.ErrInvalidAppSpec)
383 }
384 if _, overlap := spec.Secrets[fileKey]; overlap {
385 return fmt.Errorf("deployment: secret key %q collides with a secret file key in service %q: %w", fileKey, spec.Name, domain.ErrInvalidAppSpec)
386 }
387 return nil
388}
389
390// sortedEnv renders a KEY=value list in sorted key order. Values stay in
391// memory for container creation only; nothing is persisted or logged.
392func sortedEnv(envMap map[string]string) []string {
393 if len(envMap) == 0 {
394 return nil
395 }
396 keys := make([]string, 0, len(envMap))
397 for key := range envMap {
398 keys = append(keys, key)
399 }
400 sort.Strings(keys)
401 env := make([]string, 0, len(keys))
402 for _, key := range keys {
403 env = append(env, key+"="+envMap[key])
404 }
405 return env
406}
407
408// pullImage fetches the pinned image from the installation registry.
409// External refs pull anonymously; installation-registry refs use the
410// internal credentials. A missing registry config disables pulling
411// (tests and pre-pulled environments).
412func (s *Service) pullImage(ctx context.Context, image string) (string, error) {
413 if err := s.validateImageSource(image, ""); err != nil {
414 return "", err
415 }
416 if s.deps.Registry.Domain == "" {
417 return image, nil
418 }
419 if !s.deps.ImagePolicy.IsInstallationImage(image) {
420 if err := s.deps.Runtime.PullImage(ctx, image); err != nil {
421 return "", fmt.Errorf("deployment: pull image %q: %w", image, err)
422 }
423 return image, nil
424 }
425 _, remainder, ok := strings.Cut(image, "/")
426 if !ok || !strings.Contains(remainder, "@sha256:") {
427 return "", fmt.Errorf("deployment: installation image must include an exact sha256 digest: %w", domain.ErrAppImageNotAllowed)
428 }
429 pullImage := s.deps.Registry.PullAddress + "/" + remainder
430 request := domain.ImagePullRequest{Reference: pullImage, Username: s.deps.Registry.Username, Password: s.deps.Registry.Password, Transport: domain.ImagePullTransportHTTP}
431 if err := s.deps.Runtime.PullImageWithOptions(ctx, request); err != nil {
432 return "", fmt.Errorf("deployment: pull installation image %q via configured local HTTP transport %q: %w", image, s.deps.Registry.PullAddress, err)
433 }
434 digest := remainder[strings.LastIndex(remainder, "@")+1:]
435 if err := s.deps.Runtime.VerifyImageDigest(ctx, pullImage, digest); err != nil {
436 return "", fmt.Errorf("deployment: verify pulled installation image %q: %w", image, err)
437 }
438 return pullImage, nil
439}
440
441// stripImageTag removes any tag or digest suffix from an image reference.
442// colon is a registry port (localhost:15500/e2e/web:v1 strips to
443// localhost:15500/e2e/web).
444func stripImageTag(ref string) string {
445 if base, _, ok := strings.Cut(ref, "@"); ok {
446 return base
447 }
448 lastSlash := strings.LastIndex(ref, "/")
449 if idx := strings.LastIndex(ref, ":"); idx > lastSlash {
450 return ref[:idx]
451 }
452 return ref
453}