1package deployment
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "maps"
8 "sort"
9 "strings"
10
11 "github.com/bnema/gordon/internal/domain"
12)
13
14// startedCandidate is one freshly started replacement generation and its
15// observed loopback backend binds (container port -> host port).
16type startedCandidate struct {
17 Container *domain.Container
18 TCPBinds map[int]int
19 UDPBinds map[int]int
20}
21
22// createAndStart builds the runtime config and starts the replacement.
23// Preflight already pulled and inspected the pinned runtime image.
24// Every TCP-capable interface port is published on 127.0.0.1 ephemeral
25// plus every UDP interface port on 127.0.0.1/udp ephemeral: readiness
26// and the proxy dial these loopback binds rootless-first,
27// never container IPs. The runtime keeps native restarts; Gordon
28// reconciles intent at boot and in the monitor (accepted decision:
29// native runtime restarts plus daemon reconciliation).
30func (s *Service) createAndStart(ctx context.Context, app, revision string, p pinnedService, opID string, journal candidateJournal) (startedCandidate, error) {
31 // Re-resolve binds from the current policy immediately before any
32 // runtime mutation: a bind revoked since preflight must fail here,
33 // before volume ownership or container creation.
34 resolvedBinds, err := s.resolveServiceBinds(app, p.spec)
35 if err != nil {
36 return startedCandidate{}, err
37 }
38 // Re-resolve devices from the current policy under the same
39 // fail-closed rule: revoked grants never reach the runtime.
40 resolvedDevices, err := s.resolveServiceDevices(app, p.spec)
41 if err != nil {
42 return startedCandidate{}, err
43 }
44 env, err := s.serviceEnv(ctx, app, p)
45 if err != nil {
46 return startedCandidate{}, err
47 }
48 image := runtimeImageRef(p)
49 // The incarnation network is verified or created before any
50 // container exists, so a workload is only ever started on a
51 // Gordon-owned network that no other app shares.
52 appID, err := s.ensureIncarnationID(ctx, app)
53 if err != nil {
54 return startedCandidate{}, err
55 }
56 nets := s.resolveAppNetworks(appID, p.sharedNetworks)
57 if err := s.ensureAppNetworks(ctx, app, appID, nets); err != nil {
58 return startedCandidate{}, err
59 }
60 volumes := map[string]string{}
61 readOnlyVolumes := map[string]string{}
62 for _, vol := range p.spec.Volumes {
63 runtimeName := domain.RuntimeVolumeName(app, p.spec.Name, vol.Name)
64 // Durable ownership is reserved BEFORE the runtime volume
65 // exists: a crash in between leaves a protected record, never
66 // an unowned volume that prune could later adopt. The
67 // incarnation ID is already validated above, so the labels
68 // carry the same ID without another ownership load.
69 if err := s.reserveVolumeOwnership(ctx, app, appID, p.spec.Name, vol.Name, runtimeName); err != nil {
70 return startedCandidate{}, err
71 }
72 if err := s.deps.Runtime.CreateVolume(ctx, runtimeName, volumeProvenanceLabels(app, appID, p.spec.Name, revision)); err != nil {
73 // CreateVolume is idempotent at the adapter; existence was
74 // checked at preflight, so only real backend errors fail here.
75 return startedCandidate{}, fmt.Errorf("deployment: create volume %q: %w", vol.Name, err)
76 }
77 // A declared read-only mount must reach the runtime as read-only:
78 // a writable mount would let the service modify protected data.
79 if vol.ReadOnly {
80 readOnlyVolumes[vol.Path] = runtimeName
81 continue
82 }
83 volumes[vol.Path] = runtimeName
84 }
85 config := &domain.ContainerConfig{
86 Image: image,
87 Name: domain.LogicalServiceIdentity(app, p.spec.Name) + "--" + shortOp(opID),
88 Env: env,
89 Entrypoint: append([]string(nil), p.spec.Command...),
90 Volumes: volumes,
91 ReadOnlyVolumes: readOnlyVolumes,
92 Binds: resolvedBinds,
93 CDIDevices: resolvedDevices,
94 Labels: appLabels(app, p.spec.Name, revision),
95 AutoRemove: false,
96 RestartPolicy: domain.RestartPolicyAlways,
97 PortPublishes: backendPublishes(p.spec),
98 NetworkMode: nets.private,
99 Hostname: p.spec.Name,
100 Aliases: []string{p.spec.Name},
101 MemoryLimit: s.deps.Limits.MemoryBytes,
102 NanoCPUs: s.deps.Limits.NanoCPUs,
103 PidsLimit: s.deps.Limits.PidsLimit,
104 }
105 created, err := s.createContainer(ctx, p.name, config)
106 if err != nil {
107 return startedCandidate{}, err
108 }
109 // The candidate ID is durably journaled before the container is started
110 // or joins a shared network, so an interrupted deployment can be cleaned
111 // up instead of leaving an untracked running generation.
112 if err := s.recordCandidate(ctx, app, p.spec.Name, created.ID, journal); err != nil {
113 return startedCandidate{}, err
114 }
115 if err := s.connectSharedNetworks(ctx, created.ID, nets); err != nil {
116 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
117 return startedCandidate{Container: created}, err
118 }
119 if err := s.deps.Runtime.StartContainer(ctx, created.ID); err != nil {
120 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
121 return startedCandidate{Container: created}, fmt.Errorf("deployment: start container: %w", err)
122 }
123 binds, udpBinds, err := s.readBackendBinds(ctx, app, p.spec.Name, created.ID, backendPorts(p.spec))
124 if err != nil {
125 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
126 return startedCandidate{Container: created}, err
127 }
128 return startedCandidate{Container: created, TCPBinds: binds, UDPBinds: udpBinds}, nil
129}
130
131// runtimeImageRef resolves the exact runtime image reference of one pinned
132// service: the image preflight resolved, else the manifest reference pinned to
133// its digest.
134func runtimeImageRef(p pinnedService) string {
135 if p.runtimeImage != "" {
136 return p.runtimeImage
137 }
138 if p.digest == "" {
139 return p.spec.Image
140 }
141 return stripImageTag(p.spec.Image) + "@" + p.digest
142}
143
144// recordCandidate durably journals a freshly created candidate before it is
145// started. A candidate whose ID cannot be recorded must not keep running:
146// recovery could never find it, so it is removed and the failure surfaced.
147func (s *Service) recordCandidate(ctx context.Context, app, service, containerID string, journal candidateJournal) error {
148 if journal == nil {
149 return nil
150 }
151 if err := journal(ctx, containerID); err != nil {
152 retired := s.retireContainer(ctx, app, retireOptions{Service: service, Force: true}, containerID)
153 if !retired.Gone {
154 return fmt.Errorf("%w (candidate cleanup: %s)", err, cleanupDetail(retired))
155 }
156 return err
157 }
158 return nil
159}
160
161func (s *Service) createContainer(ctx context.Context, service string, config *domain.ContainerConfig) (*domain.Container, error) {
162 created, err := s.deps.Runtime.CreateContainer(ctx, config)
163 if err == nil {
164 return created, nil
165 }
166 // The engine-unsupported sentinel survives redaction so callers can
167 // map it to the structured runtime-unsupported envelope. It carries
168 // no host inventory (engine family/version only), and the device
169 // gate runs only for device-bearing creates, so this branch also
170 // covers services that declare both binds and devices.
171 if errors.Is(err, domain.ErrRuntimeUnsupported) {
172 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: %w", service, domain.ErrRuntimeUnsupported)
173 }
174 if len(config.Binds) > 0 {
175 // A CreateContainer failure is a runtime error, not a bind policy
176 // violation: policy was already enforced by resolveServiceBinds
177 // before this call. Runtime errors may embed the resolved host
178 // path, so redact the whole cause instead of mislabelling it and
179 // keep it out of operation journals and API/CLI responses.
180 return nil, fmt.Errorf("deployment: create container for service %q with administrative mounts: runtime error redacted", service)
181 }
182 if len(config.CDIDevices) > 0 {
183 // Same redaction rule for device-bearing creates: runtime errors
184 // may embed resolved CDI IDs, which are host inventory. Policy
185 // was already enforced by resolveServiceDevices before this call.
186 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: runtime error redacted", service)
187 }
188 return nil, fmt.Errorf("deployment: create container: %w", err)
189}
190
191// reserveVolumeOwnership durably records one app-owned volume as
192// attached BEFORE the runtime volume is created, so a crash between the
193// two leaves a protected record instead of an unowned volume. The
194// caller passes the validated incarnation ID from ensureIncarnationID,
195// which already assigned it, so no reload is needed for the labels.
196func (s *Service) reserveVolumeOwnership(ctx context.Context, app, appID, service, volumeName, runtimeName string) error {
197 ownership, err := s.deps.State.LoadOwnership(ctx, app)
198 if err != nil {
199 return fmt.Errorf("deployment: load ownership: %w", err)
200 }
201 if ownership.App == "" {
202 ownership.App = app
203 }
204 if ownership.ID == "" {
205 ownership.ID = appID
206 }
207 entry := domain.AppOwnedVolume{
208 Name: volumeName,
209 Service: service,
210 RuntimeName: runtimeName,
211 State: domain.AppResourceAttached,
212 }
213 replaced := false
214 for i, existing := range ownership.Volumes {
215 if existing.RuntimeName == runtimeName {
216 ownership.Volumes[i] = entry
217 replaced = true
218 break
219 }
220 }
221 if !replaced {
222 ownership.Volumes = append(ownership.Volumes, entry)
223 }
224 if err := s.deps.State.SaveOwnership(ctx, ownership); err != nil {
225 return fmt.Errorf("deployment: reserve volume ownership: %w", err)
226 }
227 return nil
228}
229
230// volumeProvenanceLabels is the label set stamped on an app-created
231// volume. Ownership is never inferred from the volume name, so the
232// labels must carry the full provenance: app, incarnation UUID, and
233// service. The managed marker is stamped here as well as merged by the
234// adapter, so provenance never depends on adapter behaviour alone.
235func volumeProvenanceLabels(app, appID, service, revision string) map[string]string {
236 labels := map[string]string{
237 domain.LabelManaged: "true",
238 domain.LabelApp: app,
239 domain.LabelAppService: service,
240 domain.LabelAppRevision: revision,
241 }
242 if appID != "" {
243 labels[domain.LabelAppID] = appID
244 }
245 return labels
246}
247
248// appLabels stamps engine ownership on every created container.
249func appLabels(app, service, revision string) map[string]string {
250 return map[string]string{
251 domain.LabelManaged: "true",
252 domain.LabelApp: app,
253 domain.LabelAppService: service,
254 domain.LabelAppRevision: revision,
255 }
256}
257
258// shortOp keeps container names short but unique per op.
259func shortOp(opID string) string {
260 trimmed := strings.TrimPrefix(opID, "op-")
261 if len(trimmed) > 12 {
262 trimmed = trimmed[:12]
263 }
264 if trimmed == "" {
265 return "op"
266 }
267 return trimmed
268}
269
270// serviceEnv resolves the service environment: the captured revision's
271// app-wide public env on every service, overlaid by that service's own
272// public env (the service value wins for the same key), plus its secret
273// values (memory only, UUID-keyed paths). Keys are emitted sorted and
274// deterministic. Overlapping public/secret keys are invalid with no
275// precedence; preflight rejects them before any mutation.
276func (s *Service) serviceEnv(ctx context.Context, app string, p pinnedService) ([]string, error) {
277 spec := p.spec
278 envMap := make(map[string]string, len(p.appEnv)+len(spec.Env)+len(spec.Secrets))
279 maps.Copy(envMap, p.appEnv)
280 maps.Copy(envMap, spec.Env)
281 keys := make([]string, 0, len(spec.Secrets))
282 for envKey := range spec.Secrets {
283 keys = append(keys, envKey)
284 }
285 sort.Strings(keys)
286 if len(keys) == 0 {
287 return sortedEnv(envMap), nil
288 }
289 id, err := s.appSecretID(ctx, app)
290 if err != nil {
291 return nil, err
292 }
293 for _, envKey := range keys {
294 if _, overlap := envMap[envKey]; overlap {
295 return nil, fmt.Errorf("deployment: env key %q collides with a secret key in service %q: %w", envKey, spec.Name, domain.ErrInvalidAppSpec)
296 }
297 path := domain.AppSecretPathForID(id, app, spec.Name, spec.Secrets[envKey])
298 value, err := s.deps.Secrets.GetSecret(ctx, path)
299 if err != nil {
300 return nil, fmt.Errorf("deployment: secret %s (env %s) missing: %w", path, envKey, domain.ErrAppSecretMissing)
301 }
302 envMap[envKey] = value
303 }
304 return sortedEnv(envMap), nil
305}
306
307// sortedEnv renders a KEY=value list in sorted key order. Values stay in
308// memory for container creation only; nothing is persisted or logged.
309func sortedEnv(envMap map[string]string) []string {
310 if len(envMap) == 0 {
311 return nil
312 }
313 keys := make([]string, 0, len(envMap))
314 for key := range envMap {
315 keys = append(keys, key)
316 }
317 sort.Strings(keys)
318 env := make([]string, 0, len(keys))
319 for _, key := range keys {
320 env = append(env, key+"="+envMap[key])
321 }
322 return env
323}
324
325// pullImage fetches the pinned image from the installation registry.
326// External refs pull anonymously; installation-registry refs use the
327// internal credentials. A missing registry config disables pulling
328// (tests and pre-pulled environments).
329func (s *Service) pullImage(ctx context.Context, image string) (string, error) {
330 if err := s.validateImageSource(image, ""); err != nil {
331 return "", err
332 }
333 if s.deps.Registry.Domain == "" {
334 return image, nil
335 }
336 if !s.deps.ImagePolicy.IsInstallationImage(image) {
337 if err := s.deps.Runtime.PullImage(ctx, image); err != nil {
338 return "", fmt.Errorf("deployment: pull image %q: %w", image, err)
339 }
340 return image, nil
341 }
342 _, remainder, ok := strings.Cut(image, "/")
343 if !ok || !strings.Contains(remainder, "@sha256:") {
344 return "", fmt.Errorf("deployment: installation image must include an exact sha256 digest: %w", domain.ErrAppImageNotAllowed)
345 }
346 pullImage := s.deps.Registry.PullAddress + "/" + remainder
347 request := domain.ImagePullRequest{Reference: pullImage, Username: s.deps.Registry.Username, Password: s.deps.Registry.Password, Transport: domain.ImagePullTransportHTTP}
348 if err := s.deps.Runtime.PullImageWithOptions(ctx, request); err != nil {
349 return "", fmt.Errorf("deployment: pull installation image %q via configured local HTTP transport %q: %w", image, s.deps.Registry.PullAddress, err)
350 }
351 digest := remainder[strings.LastIndex(remainder, "@")+1:]
352 if err := s.deps.Runtime.VerifyImageDigest(ctx, pullImage, digest); err != nil {
353 return "", fmt.Errorf("deployment: verify pulled installation image %q: %w", image, err)
354 }
355 return pullImage, nil
356}
357
358// stripImageTag removes any tag or digest suffix from an image reference.
359// colon is a registry port (localhost:15500/e2e/web:v1 strips to
360// localhost:15500/e2e/web).
361func stripImageTag(ref string) string {
362 if base, _, ok := strings.Cut(ref, "@"); ok {
363 return base
364 }
365 lastSlash := strings.LastIndex(ref, "/")
366 if idx := strings.LastIndex(ref, ":"); idx > lastSlash {
367 return ref[:idx]
368 }
369 return ref
370}