1package deployment
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "maps"
8 "slices"
9 "sort"
10 "strings"
11
12 "github.com/bnema/zerowrap"
13
14 "github.com/bnema/gordon/internal/domain"
15)
16
17// startedCandidate is one freshly started replacement generation and its
18// observed loopback backend binds (container port -> host port).
19type startedCandidate struct {
20 Container *domain.Container
21 TCPBinds map[int]int
22 UDPBinds map[int]int
23}
24
25// createAndStart builds the runtime config and starts the replacement.
26// Preflight already pulled and inspected the pinned runtime image.
27// Every TCP-capable interface port is published on 127.0.0.1 ephemeral
28// plus every UDP interface port on 127.0.0.1/udp ephemeral: readiness
29// and the proxy dial these loopback binds rootless-first,
30// never container IPs. The runtime keeps native restarts; Gordon
31// reconciles intent at boot and in the monitor (accepted decision:
32// native runtime restarts plus daemon reconciliation).
33func (s *Service) createAndStart(ctx context.Context, app, revision string, p pinnedService, opID string, journal candidateJournal) (startedCandidate, error) {
34 // Re-resolve binds from the current policy immediately before any
35 // runtime mutation: a bind revoked since preflight must fail here,
36 // before volume ownership or container creation.
37 resolvedBinds, err := s.resolveServiceBinds(app, p.spec)
38 if err != nil {
39 return startedCandidate{}, err
40 }
41 // Re-resolve devices from the current policy under the same
42 // fail-closed rule: revoked grants never reach the runtime.
43 resolvedDevices, err := s.resolveServiceDevices(app, p.spec)
44 if err != nil {
45 return startedCandidate{}, err
46 }
47 env, files, err := s.serviceEnv(ctx, app, p)
48 if err != nil {
49 return startedCandidate{}, err
50 }
51 image := runtimeImageRef(p)
52 // The incarnation network is verified or created before any
53 // container exists, so a workload is only ever started on a
54 // Gordon-owned network that no other app shares.
55 appID, err := s.ensureIncarnationID(ctx, app)
56 if err != nil {
57 return startedCandidate{}, err
58 }
59 nets := s.resolveAppNetworks(appID, p.sharedNetworks)
60 if err := s.ensureAppNetworks(ctx, app, appID, nets); err != nil {
61 return startedCandidate{}, err
62 }
63 volumes := map[string]string{}
64 readOnlyVolumes := map[string]string{}
65 for _, vol := range p.spec.Volumes {
66 runtimeName := domain.RuntimeVolumeName(app, p.spec.Name, vol.Name)
67 // Durable ownership is reserved BEFORE the runtime volume
68 // exists: a crash in between leaves a protected record, never
69 // an unowned volume that prune could later adopt. The
70 // incarnation ID is already validated above, so the labels
71 // carry the same ID without another ownership load.
72 if err := s.reserveVolumeOwnership(ctx, app, appID, p.spec.Name, vol.Name, runtimeName); err != nil {
73 return startedCandidate{}, err
74 }
75 if err := s.deps.Runtime.CreateVolume(ctx, runtimeName, volumeProvenanceLabels(app, appID, p.spec.Name, revision)); err != nil {
76 // CreateVolume is idempotent at the adapter; existence was
77 // checked at preflight, so only real backend errors fail here.
78 return startedCandidate{}, fmt.Errorf("deployment: create volume %q: %w", vol.Name, err)
79 }
80 // A declared read-only mount must reach the runtime as read-only:
81 // a writable mount would let the service modify protected data.
82 if vol.ReadOnly {
83 readOnlyVolumes[vol.Path] = runtimeName
84 continue
85 }
86 volumes[vol.Path] = runtimeName
87 }
88 config := &domain.ContainerConfig{
89 Image: image,
90 Name: domain.LogicalServiceIdentity(app, p.spec.Name) + "--" + shortOp(opID),
91 Env: env,
92 Entrypoint: append([]string(nil), p.spec.Command...),
93 Volumes: volumes,
94 ReadOnlyVolumes: readOnlyVolumes,
95 Binds: resolvedBinds,
96 CDIDevices: resolvedDevices,
97 Labels: appLabels(app, p.spec.Name, revision),
98 AutoRemove: false,
99 RestartPolicy: domain.RestartPolicyAlways,
100 PortPublishes: backendPublishes(p.spec),
101 NetworkMode: nets.private,
102 Hostname: p.spec.Name,
103 Aliases: []string{p.spec.Name},
104 MemoryLimit: s.deps.Limits.MemoryBytes,
105 NanoCPUs: s.deps.Limits.NanoCPUs,
106 PidsLimit: s.deps.Limits.PidsLimit,
107 }
108 created, err := s.createWithSecretFiles(ctx, app, p.name, config, files)
109 if err != nil {
110 return startedCandidate{}, err
111 }
112 // The candidate ID is durably journaled before the container is started
113 // or joins a shared network, so an interrupted deployment can be cleaned
114 // up instead of leaving an untracked running generation.
115 if err := s.recordCandidate(ctx, app, p.spec.Name, created.ID, journal); err != nil {
116 return startedCandidate{}, err
117 }
118 if err := s.connectSharedNetworks(ctx, created.ID, nets); err != nil {
119 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
120 return startedCandidate{Container: created}, err
121 }
122 if err := s.deps.Runtime.StartContainer(ctx, created.ID); err != nil {
123 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
124 return startedCandidate{Container: created}, fmt.Errorf("deployment: start container: %w", err)
125 }
126 binds, udpBinds, err := s.readBackendBinds(ctx, app, p.spec.Name, created.ID, backendPorts(p.spec))
127 if err != nil {
128 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
129 return startedCandidate{Container: created}, err
130 }
131 return startedCandidate{Container: created, TCPBinds: binds, UDPBinds: udpBinds}, nil
132}
133
134// runtimeImageRef resolves the exact runtime image reference of one pinned
135// service: the image preflight resolved, else the manifest reference pinned to
136// its digest.
137func runtimeImageRef(p pinnedService) string {
138 if p.runtimeImage != "" {
139 return p.runtimeImage
140 }
141 if p.digest == "" {
142 return p.spec.Image
143 }
144 return stripImageTag(p.spec.Image) + "@" + p.digest
145}
146
147// recordCandidate durably journals a freshly created candidate before it is
148// started. A candidate whose ID cannot be recorded must not keep running:
149// recovery could never find it, so it is removed and the failure surfaced.
150func (s *Service) recordCandidate(ctx context.Context, app, service, containerID string, journal candidateJournal) error {
151 if journal == nil {
152 return nil
153 }
154 if err := journal(ctx, containerID); err != nil {
155 retired := s.retireContainer(ctx, app, retireOptions{Service: service, Force: true}, containerID)
156 if !retired.Gone {
157 return fmt.Errorf("%w (candidate cleanup: %s)", err, cleanupDetail(retired))
158 }
159 return err
160 }
161 return nil
162}
163
164// createWithSecretFiles writes file-delivered secrets to the host, mounts
165// each one read-only at /run/secrets/<KEY>, and creates the container.
166// The files are removed again when the create fails. Values stay in
167// memory and on the 0444 host files only; nothing is logged.
168func (s *Service) createWithSecretFiles(ctx context.Context, app, service string, config *domain.ContainerConfig, files map[string]string) (*domain.Container, error) {
169 if len(files) == 0 {
170 return s.createContainer(ctx, service, config)
171 }
172 if s.deps.SecretFiles == nil {
173 return nil, fmt.Errorf("deployment: service %q delivers secrets as files but no secret file store is configured", service)
174 }
175 paths, err := s.deps.SecretFiles.Write(ctx, app, config.Name, files)
176 if err != nil {
177 return nil, fmt.Errorf("deployment: write secret files for service %q: %w", service, err)
178 }
179 withFiles := *config
180 withFiles.Binds = append([]domain.ContainerBind(nil), config.Binds...)
181 for _, key := range slices.Sorted(maps.Keys(files)) {
182 withFiles.Binds = append(withFiles.Binds, domain.ContainerBind{
183 Name: "secret-file-" + key,
184 Source: paths[key],
185 Destination: domain.SecretFilePath(key),
186 ReadOnly: true,
187 })
188 }
189 created, err := s.createContainer(ctx, service, &withFiles)
190 if err != nil {
191 if rmErr := s.deps.SecretFiles.Remove(ctx, app, config.Name); rmErr != nil {
192 log := zerowrap.FromCtx(ctx)
193 log.Warn().Str("app", app).Str("service", service).
194 Msg("deployment: secret files cleanup after failed create failed")
195 }
196 return nil, err
197 }
198 return created, nil
199}
200
201func (s *Service) createContainer(ctx context.Context, service string, config *domain.ContainerConfig) (*domain.Container, error) {
202 if err := s.deps.Guard.Check(config); err != nil {
203 return nil, fmt.Errorf("deployment: create container for service %q: %w", service, err)
204 }
205 created, err := s.deps.Runtime.CreateContainer(ctx, config)
206 if err == nil {
207 return created, nil
208 }
209 // The engine-unsupported sentinel survives redaction so callers can
210 // map it to the structured runtime-unsupported envelope. It carries
211 // no host inventory (engine family/version only), and the device
212 // gate runs only for device-bearing creates, so this branch also
213 // covers services that declare both binds and devices.
214 if errors.Is(err, domain.ErrRuntimeUnsupported) {
215 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: %w", service, domain.ErrRuntimeUnsupported)
216 }
217 if len(config.Binds) > 0 {
218 // A CreateContainer failure is a runtime error, not a bind policy
219 // violation: policy was already enforced by resolveServiceBinds
220 // before this call. Runtime errors may embed the resolved host
221 // path, so redact the whole cause instead of mislabelling it and
222 // keep it out of operation journals and API/CLI responses.
223 return nil, fmt.Errorf("deployment: create container for service %q with administrative mounts: runtime error redacted", service)
224 }
225 if len(config.CDIDevices) > 0 {
226 // Same redaction rule for device-bearing creates: runtime errors
227 // may embed resolved CDI IDs, which are host inventory. Policy
228 // was already enforced by resolveServiceDevices before this call.
229 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: runtime error redacted", service)
230 }
231 return nil, fmt.Errorf("deployment: create container: %w", err)
232}
233
234// reserveVolumeOwnership durably records one app-owned volume as
235// attached BEFORE the runtime volume is created, so a crash between the
236// two leaves a protected record instead of an unowned volume. The
237// caller passes the validated incarnation ID from ensureIncarnationID,
238// which already assigned it, so no reload is needed for the labels.
239func (s *Service) reserveVolumeOwnership(ctx context.Context, app, appID, service, volumeName, runtimeName string) error {
240 ownership, err := s.deps.State.LoadOwnership(ctx, app)
241 if err != nil {
242 return fmt.Errorf("deployment: load ownership: %w", err)
243 }
244 if ownership.App == "" {
245 ownership.App = app
246 }
247 if ownership.ID == "" {
248 ownership.ID = appID
249 }
250 entry := domain.AppOwnedVolume{
251 Name: volumeName,
252 Service: service,
253 RuntimeName: runtimeName,
254 State: domain.AppResourceAttached,
255 }
256 replaced := false
257 for i, existing := range ownership.Volumes {
258 if existing.RuntimeName == runtimeName {
259 ownership.Volumes[i] = entry
260 replaced = true
261 break
262 }
263 }
264 if !replaced {
265 ownership.Volumes = append(ownership.Volumes, entry)
266 }
267 if err := s.deps.State.SaveOwnership(ctx, ownership); err != nil {
268 return fmt.Errorf("deployment: reserve volume ownership: %w", err)
269 }
270 return nil
271}
272
273// volumeProvenanceLabels is the label set stamped on an app-created
274// volume. Ownership is never inferred from the volume name, so the
275// labels must carry the full provenance: app, incarnation UUID, and
276// service. The managed marker is stamped here as well as merged by the
277// adapter, so provenance never depends on adapter behaviour alone.
278func volumeProvenanceLabels(app, appID, service, revision string) map[string]string {
279 labels := map[string]string{
280 domain.LabelManaged: "true",
281 domain.LabelApp: app,
282 domain.LabelAppService: service,
283 domain.LabelAppRevision: revision,
284 }
285 if appID != "" {
286 labels[domain.LabelAppID] = appID
287 }
288 return labels
289}
290
291// appLabels stamps engine ownership on every created container.
292func appLabels(app, service, revision string) map[string]string {
293 return map[string]string{
294 domain.LabelManaged: "true",
295 domain.LabelApp: app,
296 domain.LabelAppService: service,
297 domain.LabelAppRevision: revision,
298 }
299}
300
301// shortOp keeps container names short but unique per op.
302func shortOp(opID string) string {
303 trimmed := strings.TrimPrefix(opID, "op-")
304 if len(trimmed) > 12 {
305 trimmed = trimmed[:12]
306 }
307 if trimmed == "" {
308 return "op"
309 }
310 return trimmed
311}
312
313// serviceEnv resolves the service environment: the captured revision's
314// app-wide public env on every service, overlaid by that service's own
315// public env (the service value wins for the same key), plus its secret
316// values (memory only, UUID-keyed paths). Keys are emitted sorted and
317// deterministic. Overlapping public/secret keys are invalid with no
318// precedence; preflight rejects them before any mutation.
319//
320// File-delivered secrets are returned in files (env key -> value, nil
321// when none) and the env gets <KEY>_FILE=/run/secrets/<KEY> instead.
322func (s *Service) serviceEnv(ctx context.Context, app string, p pinnedService) ([]string, map[string]string, error) {
323 spec := p.spec
324 envMap := make(map[string]string, len(p.appEnv)+len(spec.Env)+len(spec.Secrets))
325 maps.Copy(envMap, p.appEnv)
326 maps.Copy(envMap, spec.Env)
327 keys := make([]string, 0, len(spec.Secrets))
328 for envKey := range spec.Secrets {
329 keys = append(keys, envKey)
330 }
331 sort.Strings(keys)
332 if len(keys) == 0 {
333 return sortedEnv(envMap), nil, nil
334 }
335 id, err := s.appSecretID(ctx, app)
336 if err != nil {
337 return nil, nil, err
338 }
339 var files map[string]string
340 for _, envKey := range keys {
341 if err := checkSecretEnvOverlap(envMap, spec, envKey); err != nil {
342 return nil, nil, err
343 }
344 path := domain.AppSecretPathForID(id, app, spec.Name, spec.Secrets[envKey])
345 value, err := s.deps.Secrets.GetSecret(ctx, path)
346 if err != nil {
347 return nil, nil, fmt.Errorf("deployment: secret %s (env %s) missing: %w", path, envKey, domain.ErrAppSecretMissing)
348 }
349 if !spec.SecretAsFile(envKey) {
350 envMap[envKey] = value
351 continue
352 }
353 if files == nil {
354 files = make(map[string]string, len(spec.SecretFiles))
355 }
356 files[envKey] = value
357 envMap[domain.SecretFileEnvKey(envKey)] = domain.SecretFilePath(envKey)
358 }
359 return sortedEnv(envMap), files, nil
360}
361
362// checkSecretEnvOverlap refuses a secret key (and, for file delivery, its
363// <KEY>_FILE key) that is already set by public env or another secret.
364func checkSecretEnvOverlap(envMap map[string]string, spec domain.AppService, envKey string) error {
365 if _, overlap := envMap[envKey]; overlap {
366 return fmt.Errorf("deployment: env key %q collides with a secret key in service %q: %w", envKey, spec.Name, domain.ErrInvalidAppSpec)
367 }
368 if !spec.SecretAsFile(envKey) {
369 return nil
370 }
371 fileKey := domain.SecretFileEnvKey(envKey)
372 if _, overlap := envMap[fileKey]; overlap {
373 return fmt.Errorf("deployment: env key %q collides with a secret file key in service %q: %w", fileKey, spec.Name, domain.ErrInvalidAppSpec)
374 }
375 if _, overlap := spec.Secrets[fileKey]; overlap {
376 return fmt.Errorf("deployment: secret key %q collides with a secret file key in service %q: %w", fileKey, spec.Name, domain.ErrInvalidAppSpec)
377 }
378 return nil
379}
380
381// sortedEnv renders a KEY=value list in sorted key order. Values stay in
382// memory for container creation only; nothing is persisted or logged.
383func sortedEnv(envMap map[string]string) []string {
384 if len(envMap) == 0 {
385 return nil
386 }
387 keys := make([]string, 0, len(envMap))
388 for key := range envMap {
389 keys = append(keys, key)
390 }
391 sort.Strings(keys)
392 env := make([]string, 0, len(keys))
393 for _, key := range keys {
394 env = append(env, key+"="+envMap[key])
395 }
396 return env
397}
398
399// pullImage fetches the pinned image from the installation registry.
400// External refs pull anonymously; installation-registry refs use the
401// internal credentials. A missing registry config disables pulling
402// (tests and pre-pulled environments).
403func (s *Service) pullImage(ctx context.Context, image string) (string, error) {
404 if err := s.validateImageSource(image, ""); err != nil {
405 return "", err
406 }
407 if s.deps.Registry.Domain == "" {
408 return image, nil
409 }
410 if !s.deps.ImagePolicy.IsInstallationImage(image) {
411 if err := s.deps.Runtime.PullImage(ctx, image); err != nil {
412 return "", fmt.Errorf("deployment: pull image %q: %w", image, err)
413 }
414 return image, nil
415 }
416 _, remainder, ok := strings.Cut(image, "/")
417 if !ok || !strings.Contains(remainder, "@sha256:") {
418 return "", fmt.Errorf("deployment: installation image must include an exact sha256 digest: %w", domain.ErrAppImageNotAllowed)
419 }
420 pullImage := s.deps.Registry.PullAddress + "/" + remainder
421 request := domain.ImagePullRequest{Reference: pullImage, Username: s.deps.Registry.Username, Password: s.deps.Registry.Password, Transport: domain.ImagePullTransportHTTP}
422 if err := s.deps.Runtime.PullImageWithOptions(ctx, request); err != nil {
423 return "", fmt.Errorf("deployment: pull installation image %q via configured local HTTP transport %q: %w", image, s.deps.Registry.PullAddress, err)
424 }
425 digest := remainder[strings.LastIndex(remainder, "@")+1:]
426 if err := s.deps.Runtime.VerifyImageDigest(ctx, pullImage, digest); err != nil {
427 return "", fmt.Errorf("deployment: verify pulled installation image %q: %w", image, err)
428 }
429 return pullImage, nil
430}
431
432// stripImageTag removes any tag or digest suffix from an image reference.
433// colon is a registry port (localhost:15500/e2e/web:v1 strips to
434// localhost:15500/e2e/web).
435func stripImageTag(ref string) string {
436 if base, _, ok := strings.Cut(ref, "@"); ok {
437 return base
438 }
439 lastSlash := strings.LastIndex(ref, "/")
440 if idx := strings.LastIndex(ref, ":"); idx > lastSlash {
441 return ref[:idx]
442 }
443 return ref
444}