1package deployment
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "maps"
8 "slices"
9 "sort"
10 "strings"
11 "time"
12
13 "github.com/bnema/zerowrap"
14
15 "github.com/bnema/gordon/internal/domain"
16)
17
18// startedCandidate is one freshly started replacement generation and its
19// observed loopback backend binds (container port -> host port).
20type startedCandidate struct {
21 Container *domain.Container
22 TCPBinds map[int]int
23 UDPBinds map[int]int
24 // StartedAt is the execution start observed right after the container
25 // was started, before readiness ran. Zero when it could not be read.
26 StartedAt time.Time
27}
28
29// createAndStart builds the runtime config and starts the replacement.
30// Preflight already pulled and inspected the pinned runtime image.
31// Every TCP-capable interface port is published on 127.0.0.1 ephemeral
32// plus every UDP interface port on 127.0.0.1/udp ephemeral: readiness
33// and the proxy dial these loopback binds rootless-first,
34// never container IPs. The runtime keeps native restarts; Gordon
35// reconciles intent at boot and in the monitor (accepted decision:
36// native runtime restarts plus daemon reconciliation).
37func (s *Service) createAndStart(ctx context.Context, app, revision string, p pinnedService, opID string, journal candidateJournal) (startedCandidate, error) {
38 // Re-resolve binds from the current policy immediately before any
39 // runtime mutation: a bind revoked since preflight must fail here,
40 // before volume ownership or container creation.
41 resolvedBinds, err := s.resolveServiceBinds(app, p.spec)
42 if err != nil {
43 return startedCandidate{}, err
44 }
45 // Re-resolve devices from the current policy under the same
46 // fail-closed rule: revoked grants never reach the runtime.
47 resolvedDevices, err := s.resolveServiceDevices(app, p.spec)
48 if err != nil {
49 return startedCandidate{}, err
50 }
51 env, files, err := s.serviceEnv(ctx, app, p)
52 if err != nil {
53 return startedCandidate{}, err
54 }
55 image := runtimeImageRef(p)
56 // The incarnation network is verified or created before any
57 // container exists, so a workload is only ever started on a
58 // Gordon-owned network that no other app shares.
59 appID, err := s.ensureIncarnationID(ctx, app)
60 if err != nil {
61 return startedCandidate{}, err
62 }
63 nets := s.resolveAppNetworks(appID, p.sharedNetworks)
64 if err := s.ensureAppNetworks(ctx, app, appID, nets); err != nil {
65 return startedCandidate{}, err
66 }
67 volumes := map[string]string{}
68 readOnlyVolumes := map[string]string{}
69 for _, vol := range p.spec.Volumes {
70 runtimeName := domain.RuntimeVolumeName(app, p.spec.Name, vol.Name)
71 // Durable ownership is reserved BEFORE the runtime volume
72 // exists: a crash in between leaves a protected record, never
73 // an unowned volume that prune could later adopt. The
74 // incarnation ID is already validated above, so the labels
75 // carry the same ID without another ownership load.
76 if err := s.reserveVolumeOwnership(ctx, app, appID, p.spec.Name, vol.Name, runtimeName); err != nil {
77 return startedCandidate{}, err
78 }
79 if err := s.deps.Runtime.CreateVolume(ctx, runtimeName, volumeProvenanceLabels(app, appID, p.spec.Name, revision)); err != nil {
80 // CreateVolume is idempotent at the adapter; existence was
81 // checked at preflight, so only real backend errors fail here.
82 return startedCandidate{}, fmt.Errorf("deployment: create volume %q: %w", vol.Name, err)
83 }
84 // A declared read-only mount must reach the runtime as read-only:
85 // a writable mount would let the service modify protected data.
86 if vol.ReadOnly {
87 readOnlyVolumes[vol.Path] = runtimeName
88 continue
89 }
90 volumes[vol.Path] = runtimeName
91 }
92 config := &domain.ContainerConfig{
93 Image: image,
94 Name: domain.LogicalServiceIdentity(app, p.spec.Name) + "--" + shortOp(opID),
95 Env: env,
96 Entrypoint: append([]string(nil), p.spec.Command...),
97 Volumes: volumes,
98 ReadOnlyVolumes: readOnlyVolumes,
99 Binds: resolvedBinds,
100 CDIDevices: resolvedDevices,
101 Labels: appLabels(app, p.spec.Name, revision),
102 AutoRemove: false,
103 RestartPolicy: domain.RestartPolicyAlways,
104 PortPublishes: backendPublishes(p.spec),
105 NetworkMode: nets.private,
106 Hostname: p.spec.Name,
107 Aliases: []string{p.spec.Name},
108 MemoryLimit: s.deps.Limits.MemoryBytes,
109 NanoCPUs: s.deps.Limits.NanoCPUs,
110 PidsLimit: s.deps.Limits.PidsLimit,
111 }
112 created, err := s.createWithSecretFiles(ctx, app, p.name, config, files)
113 if err != nil {
114 return startedCandidate{}, err
115 }
116 // The candidate ID is durably journaled before the container is started
117 // or joins a shared network, so an interrupted deployment can be cleaned
118 // up instead of leaving an untracked running generation.
119 if err := s.recordCandidate(ctx, app, p.spec.Name, created.ID, journal); err != nil {
120 return startedCandidate{}, err
121 }
122 if err := s.connectSharedNetworks(ctx, created.ID, nets); err != nil {
123 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
124 return startedCandidate{Container: created}, err
125 }
126 if err := s.deps.Runtime.StartContainer(ctx, created.ID); err != nil {
127 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
128 return startedCandidate{Container: created}, fmt.Errorf("deployment: start container: %w", err)
129 }
130 // The execution is observed before readiness runs: a native restart
131 // between readiness and the verified-execution record then shows up as a
132 // different start time, so it is still treated as a fresh execution.
133 startedAt := s.executionStart(ctx, created.ID)
134 binds, udpBinds, err := s.readBackendBinds(ctx, app, p.spec.Name, created.ID, backendPorts(p.spec))
135 if err != nil {
136 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
137 return startedCandidate{Container: created}, err
138 }
139 return startedCandidate{Container: created, TCPBinds: binds, UDPBinds: udpBinds, StartedAt: startedAt}, nil
140}
141
142// runtimeImageRef resolves the exact runtime image reference of one pinned
143// service: the image preflight resolved, else the manifest reference pinned to
144// its digest.
145func runtimeImageRef(p pinnedService) string {
146 if p.runtimeImage != "" {
147 return p.runtimeImage
148 }
149 if p.digest == "" {
150 return p.spec.Image
151 }
152 return stripImageTag(p.spec.Image) + "@" + p.digest
153}
154
155// recordCandidate durably journals a freshly created candidate before it is
156// started. A candidate whose ID cannot be recorded must not keep running:
157// recovery could never find it, so it is removed and the failure surfaced.
158func (s *Service) recordCandidate(ctx context.Context, app, service, containerID string, journal candidateJournal) error {
159 if journal == nil {
160 return nil
161 }
162 if err := journal(ctx, containerID); err != nil {
163 retired := s.retireContainer(ctx, app, retireOptions{Service: service, Force: true}, containerID)
164 if !retired.Gone {
165 return fmt.Errorf("%w (candidate cleanup: %s)", err, cleanupDetail(retired))
166 }
167 return err
168 }
169 return nil
170}
171
172// createWithSecretFiles writes file-delivered secrets to the host, mounts
173// each one read-only at /run/secrets/<KEY>, and creates the container.
174// The files are removed again when the create fails. Values stay in
175// memory and on the 0444 host files only; nothing is logged.
176func (s *Service) createWithSecretFiles(ctx context.Context, app, service string, config *domain.ContainerConfig, files map[string]string) (*domain.Container, error) {
177 if len(files) == 0 {
178 return s.createContainer(ctx, service, config)
179 }
180 if s.deps.SecretFiles == nil {
181 return nil, fmt.Errorf("deployment: service %q delivers secrets as files but no secret file store is configured", service)
182 }
183 paths, err := s.deps.SecretFiles.Write(ctx, app, config.Name, files)
184 if err != nil {
185 return nil, fmt.Errorf("deployment: write secret files for service %q: %w", service, err)
186 }
187 withFiles := *config
188 withFiles.Binds = append([]domain.ContainerBind(nil), config.Binds...)
189 for _, key := range slices.Sorted(maps.Keys(files)) {
190 withFiles.Binds = append(withFiles.Binds, domain.ContainerBind{
191 Name: "secret-file-" + key,
192 Source: paths[key],
193 Destination: domain.SecretFilePath(key),
194 ReadOnly: true,
195 })
196 }
197 created, err := s.createContainer(ctx, service, &withFiles)
198 if err != nil {
199 if rmErr := s.deps.SecretFiles.Remove(ctx, app, config.Name); rmErr != nil {
200 log := zerowrap.FromCtx(ctx)
201 log.Warn().Str("app", app).Str("service", service).
202 Msg("deployment: secret files cleanup after failed create failed")
203 }
204 return nil, err
205 }
206 return created, nil
207}
208
209func (s *Service) createContainer(ctx context.Context, service string, config *domain.ContainerConfig) (*domain.Container, error) {
210 if err := s.deps.Guard.Check(config); err != nil {
211 return nil, fmt.Errorf("deployment: create container for service %q: %w", service, err)
212 }
213 created, err := s.deps.Runtime.CreateContainer(ctx, config)
214 if err == nil {
215 return created, nil
216 }
217 // The engine-unsupported sentinel survives redaction so callers can
218 // map it to the structured runtime-unsupported envelope. It carries
219 // no host inventory (engine family/version only), and the device
220 // gate runs only for device-bearing creates, so this branch also
221 // covers services that declare both binds and devices.
222 if errors.Is(err, domain.ErrRuntimeUnsupported) {
223 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: %w", service, domain.ErrRuntimeUnsupported)
224 }
225 if len(config.Binds) > 0 {
226 // A CreateContainer failure is a runtime error, not a bind policy
227 // violation: policy was already enforced by resolveServiceBinds
228 // before this call. Runtime errors may embed the resolved host
229 // path, so redact the whole cause instead of mislabelling it and
230 // keep it out of operation journals and API/CLI responses.
231 return nil, fmt.Errorf("deployment: create container for service %q with administrative mounts: runtime error redacted", service)
232 }
233 if len(config.CDIDevices) > 0 {
234 // Same redaction rule for device-bearing creates: runtime errors
235 // may embed resolved CDI IDs, which are host inventory. Policy
236 // was already enforced by resolveServiceDevices before this call.
237 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: runtime error redacted", service)
238 }
239 return nil, fmt.Errorf("deployment: create container: %w", err)
240}
241
242// reserveVolumeOwnership durably records one app-owned volume as
243// attached BEFORE the runtime volume is created, so a crash between the
244// two leaves a protected record instead of an unowned volume. The
245// caller passes the validated incarnation ID from ensureIncarnationID,
246// which already assigned it, so no reload is needed for the labels.
247func (s *Service) reserveVolumeOwnership(ctx context.Context, app, appID, service, volumeName, runtimeName string) error {
248 ownership, err := s.deps.State.LoadOwnership(ctx, app)
249 if err != nil {
250 return fmt.Errorf("deployment: load ownership: %w", err)
251 }
252 if ownership.App == "" {
253 ownership.App = app
254 }
255 if ownership.ID == "" {
256 ownership.ID = appID
257 }
258 entry := domain.AppOwnedVolume{
259 Name: volumeName,
260 Service: service,
261 RuntimeName: runtimeName,
262 State: domain.AppResourceAttached,
263 }
264 replaced := false
265 for i, existing := range ownership.Volumes {
266 if existing.RuntimeName == runtimeName {
267 ownership.Volumes[i] = entry
268 replaced = true
269 break
270 }
271 }
272 if !replaced {
273 ownership.Volumes = append(ownership.Volumes, entry)
274 }
275 if err := s.deps.State.SaveOwnership(ctx, ownership); err != nil {
276 return fmt.Errorf("deployment: reserve volume ownership: %w", err)
277 }
278 return nil
279}
280
281// volumeProvenanceLabels is the label set stamped on an app-created
282// volume. Ownership is never inferred from the volume name, so the
283// labels must carry the full provenance: app, incarnation UUID, and
284// service. The managed marker is stamped here as well as merged by the
285// adapter, so provenance never depends on adapter behaviour alone.
286func volumeProvenanceLabels(app, appID, service, revision string) map[string]string {
287 labels := map[string]string{
288 domain.LabelManaged: "true",
289 domain.LabelApp: app,
290 domain.LabelAppService: service,
291 domain.LabelAppRevision: revision,
292 }
293 if appID != "" {
294 labels[domain.LabelAppID] = appID
295 }
296 return labels
297}
298
299// appLabels stamps engine ownership on every created container.
300func appLabels(app, service, revision string) map[string]string {
301 return map[string]string{
302 domain.LabelManaged: "true",
303 domain.LabelApp: app,
304 domain.LabelAppService: service,
305 domain.LabelAppRevision: revision,
306 }
307}
308
309// shortOp keeps container names short but unique per op.
310func shortOp(opID string) string {
311 trimmed := strings.TrimPrefix(opID, "op-")
312 if len(trimmed) > 12 {
313 trimmed = trimmed[:12]
314 }
315 if trimmed == "" {
316 return "op"
317 }
318 return trimmed
319}
320
321// serviceEnv resolves the service environment: the captured revision's
322// app-wide public env on every service, overlaid by that service's own
323// public env (the service value wins for the same key), plus its secret
324// values (memory only, UUID-keyed paths). Keys are emitted sorted and
325// deterministic. Overlapping public/secret keys are invalid with no
326// precedence; preflight rejects them before any mutation.
327//
328// File-delivered secrets are returned in files (env key -> value, nil
329// when none) and the env gets <KEY>_FILE=/run/secrets/<KEY> instead.
330func (s *Service) serviceEnv(ctx context.Context, app string, p pinnedService) ([]string, map[string]string, error) {
331 spec := p.spec
332 envMap := make(map[string]string, len(p.appEnv)+len(spec.Env)+len(spec.Secrets))
333 maps.Copy(envMap, p.appEnv)
334 maps.Copy(envMap, spec.Env)
335 keys := make([]string, 0, len(spec.Secrets))
336 for envKey := range spec.Secrets {
337 keys = append(keys, envKey)
338 }
339 sort.Strings(keys)
340 if len(keys) == 0 {
341 return sortedEnv(envMap), nil, nil
342 }
343 id, err := s.appSecretID(ctx, app)
344 if err != nil {
345 return nil, nil, err
346 }
347 var files map[string]string
348 for _, envKey := range keys {
349 if err := checkSecretEnvOverlap(envMap, spec, envKey); err != nil {
350 return nil, nil, err
351 }
352 path := domain.AppSecretPathForID(id, app, spec.Name, spec.Secrets[envKey])
353 value, err := s.deps.Secrets.GetSecret(ctx, path)
354 if err != nil {
355 return nil, nil, fmt.Errorf("deployment: secret %s (env %s) missing: %w", path, envKey, domain.ErrAppSecretMissing)
356 }
357 if !spec.SecretAsFile(envKey) {
358 envMap[envKey] = value
359 continue
360 }
361 if files == nil {
362 files = make(map[string]string, len(spec.SecretFiles))
363 }
364 files[envKey] = value
365 envMap[domain.SecretFileEnvKey(envKey)] = domain.SecretFilePath(envKey)
366 }
367 return sortedEnv(envMap), files, nil
368}
369
370// checkSecretEnvOverlap refuses a secret key (and, for file delivery, its
371// <KEY>_FILE key) that is already set by public env or another secret.
372func checkSecretEnvOverlap(envMap map[string]string, spec domain.AppService, envKey string) error {
373 if _, overlap := envMap[envKey]; overlap {
374 return fmt.Errorf("deployment: env key %q collides with a secret key in service %q: %w", envKey, spec.Name, domain.ErrInvalidAppSpec)
375 }
376 if !spec.SecretAsFile(envKey) {
377 return nil
378 }
379 fileKey := domain.SecretFileEnvKey(envKey)
380 if _, overlap := envMap[fileKey]; overlap {
381 return fmt.Errorf("deployment: env key %q collides with a secret file key in service %q: %w", fileKey, spec.Name, domain.ErrInvalidAppSpec)
382 }
383 if _, overlap := spec.Secrets[fileKey]; overlap {
384 return fmt.Errorf("deployment: secret key %q collides with a secret file key in service %q: %w", fileKey, spec.Name, domain.ErrInvalidAppSpec)
385 }
386 return nil
387}
388
389// sortedEnv renders a KEY=value list in sorted key order. Values stay in
390// memory for container creation only; nothing is persisted or logged.
391func sortedEnv(envMap map[string]string) []string {
392 if len(envMap) == 0 {
393 return nil
394 }
395 keys := make([]string, 0, len(envMap))
396 for key := range envMap {
397 keys = append(keys, key)
398 }
399 sort.Strings(keys)
400 env := make([]string, 0, len(keys))
401 for _, key := range keys {
402 env = append(env, key+"="+envMap[key])
403 }
404 return env
405}
406
407// pullImage fetches the pinned image from the installation registry.
408// External refs pull anonymously; installation-registry refs use the
409// internal credentials. A missing registry config disables pulling
410// (tests and pre-pulled environments).
411func (s *Service) pullImage(ctx context.Context, image string) (string, error) {
412 if err := s.validateImageSource(image, ""); err != nil {
413 return "", err
414 }
415 if s.deps.Registry.Domain == "" {
416 return image, nil
417 }
418 if !s.deps.ImagePolicy.IsInstallationImage(image) {
419 if err := s.deps.Runtime.PullImage(ctx, image); err != nil {
420 return "", fmt.Errorf("deployment: pull image %q: %w", image, err)
421 }
422 return image, nil
423 }
424 _, remainder, ok := strings.Cut(image, "/")
425 if !ok || !strings.Contains(remainder, "@sha256:") {
426 return "", fmt.Errorf("deployment: installation image must include an exact sha256 digest: %w", domain.ErrAppImageNotAllowed)
427 }
428 pullImage := s.deps.Registry.PullAddress + "/" + remainder
429 request := domain.ImagePullRequest{Reference: pullImage, Username: s.deps.Registry.Username, Password: s.deps.Registry.Password, Transport: domain.ImagePullTransportHTTP}
430 if err := s.deps.Runtime.PullImageWithOptions(ctx, request); err != nil {
431 return "", fmt.Errorf("deployment: pull installation image %q via configured local HTTP transport %q: %w", image, s.deps.Registry.PullAddress, err)
432 }
433 digest := remainder[strings.LastIndex(remainder, "@")+1:]
434 if err := s.deps.Runtime.VerifyImageDigest(ctx, pullImage, digest); err != nil {
435 return "", fmt.Errorf("deployment: verify pulled installation image %q: %w", image, err)
436 }
437 return pullImage, nil
438}
439
440// stripImageTag removes any tag or digest suffix from an image reference.
441// colon is a registry port (localhost:15500/e2e/web:v1 strips to
442// localhost:15500/e2e/web).
443func stripImageTag(ref string) string {
444 if base, _, ok := strings.Cut(ref, "@"); ok {
445 return base
446 }
447 lastSlash := strings.LastIndex(ref, "/")
448 if idx := strings.LastIndex(ref, ":"); idx > lastSlash {
449 return ref[:idx]
450 }
451 return ref
452}