1package deployment
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "sort"
8 "strings"
9
10 "github.com/bnema/gordon/internal/domain"
11)
12
13// startedCandidate is one freshly started replacement generation and its
14// observed loopback backend binds (container port -> host port).
15type startedCandidate struct {
16 Container *domain.Container
17 TCPBinds map[int]int
18 UDPBinds map[int]int
19}
20
21// createAndStart builds the runtime config and starts the replacement.
22// Preflight already pulled and inspected the pinned runtime image.
23// Every TCP-capable interface port is published on 127.0.0.1 ephemeral
24// plus every UDP interface port on 127.0.0.1/udp ephemeral: readiness
25// and the proxy dial these loopback binds rootless-first,
26// never container IPs. The runtime keeps native restarts; Gordon
27// reconciles intent at boot and in the monitor (accepted decision:
28// native runtime restarts plus daemon reconciliation).
29func (s *Service) createAndStart(ctx context.Context, app, revision string, p pinnedService, opID string, journal candidateJournal) (startedCandidate, error) {
30 // Re-resolve binds from the current policy immediately before any
31 // runtime mutation: a bind revoked since preflight must fail here,
32 // before volume ownership or container creation.
33 resolvedBinds, err := s.resolveServiceBinds(app, p.spec)
34 if err != nil {
35 return startedCandidate{}, err
36 }
37 // Re-resolve devices from the current policy under the same
38 // fail-closed rule: revoked grants never reach the runtime.
39 resolvedDevices, err := s.resolveServiceDevices(app, p.spec)
40 if err != nil {
41 return startedCandidate{}, err
42 }
43 env, err := s.serviceEnv(ctx, app, p)
44 if err != nil {
45 return startedCandidate{}, err
46 }
47 image := runtimeImageRef(p)
48 // The incarnation network is verified or created before any
49 // container exists, so a workload is only ever started on a
50 // Gordon-owned network that no other app shares.
51 appID, err := s.ensureIncarnationID(ctx, app)
52 if err != nil {
53 return startedCandidate{}, err
54 }
55 nets := s.resolveAppNetworks(appID, p.sharedNetworks)
56 if err := s.ensureAppNetworks(ctx, app, appID, nets); err != nil {
57 return startedCandidate{}, err
58 }
59 volumes := map[string]string{}
60 readOnlyVolumes := map[string]string{}
61 for _, vol := range p.spec.Volumes {
62 runtimeName := domain.RuntimeVolumeName(app, p.spec.Name, vol.Name)
63 // Durable ownership is reserved BEFORE the runtime volume
64 // exists: a crash in between leaves a protected record, never
65 // an unowned volume that prune could later adopt. The
66 // incarnation ID is already validated above, so the labels
67 // carry the same ID without another ownership load.
68 if err := s.reserveVolumeOwnership(ctx, app, appID, p.spec.Name, vol.Name, runtimeName); err != nil {
69 return startedCandidate{}, err
70 }
71 if err := s.deps.Runtime.CreateVolume(ctx, runtimeName, volumeProvenanceLabels(app, appID, p.spec.Name, revision)); err != nil {
72 // CreateVolume is idempotent at the adapter; existence was
73 // checked at preflight, so only real backend errors fail here.
74 return startedCandidate{}, fmt.Errorf("deployment: create volume %q: %w", vol.Name, err)
75 }
76 // A declared read-only mount must reach the runtime as read-only:
77 // a writable mount would let the service modify protected data.
78 if vol.ReadOnly {
79 readOnlyVolumes[vol.Path] = runtimeName
80 continue
81 }
82 volumes[vol.Path] = runtimeName
83 }
84 config := &domain.ContainerConfig{
85 Image: image,
86 Name: domain.LogicalServiceIdentity(app, p.spec.Name) + "--" + shortOp(opID),
87 Env: env,
88 Entrypoint: append([]string(nil), p.spec.Command...),
89 Volumes: volumes,
90 ReadOnlyVolumes: readOnlyVolumes,
91 Binds: resolvedBinds,
92 CDIDevices: resolvedDevices,
93 Labels: appLabels(app, p.spec.Name, revision),
94 AutoRemove: false,
95 RestartPolicy: domain.RestartPolicyAlways,
96 PortPublishes: backendPublishes(p.spec),
97 NetworkMode: nets.private,
98 Hostname: p.spec.Name,
99 Aliases: []string{p.spec.Name},
100 MemoryLimit: s.deps.Limits.MemoryBytes,
101 NanoCPUs: s.deps.Limits.NanoCPUs,
102 PidsLimit: s.deps.Limits.PidsLimit,
103 }
104 created, err := s.createContainer(ctx, p.name, config)
105 if err != nil {
106 return startedCandidate{}, err
107 }
108 // The candidate ID is durably journaled before the container is started
109 // or joins a shared network, so an interrupted deployment can be cleaned
110 // up instead of leaving an untracked running generation.
111 if err := s.recordCandidate(ctx, app, p.spec.Name, created.ID, journal); err != nil {
112 return startedCandidate{}, err
113 }
114 if err := s.connectSharedNetworks(ctx, created.ID, nets); err != nil {
115 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
116 return startedCandidate{Container: created}, err
117 }
118 if err := s.deps.Runtime.StartContainer(ctx, created.ID); err != nil {
119 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
120 return startedCandidate{Container: created}, fmt.Errorf("deployment: start container: %w", err)
121 }
122 binds, udpBinds, err := s.readBackendBinds(ctx, app, p.spec.Name, created.ID, backendPorts(p.spec))
123 if err != nil {
124 s.retireCandidate(ctx, app, p.spec.Name, created.ID)
125 return startedCandidate{Container: created}, err
126 }
127 return startedCandidate{Container: created, TCPBinds: binds, UDPBinds: udpBinds}, nil
128}
129
130// runtimeImageRef resolves the exact runtime image reference of one pinned
131// service: the image preflight resolved, else the manifest reference pinned to
132// its digest.
133func runtimeImageRef(p pinnedService) string {
134 if p.runtimeImage != "" {
135 return p.runtimeImage
136 }
137 if p.digest == "" {
138 return p.spec.Image
139 }
140 return stripImageTag(p.spec.Image) + "@" + p.digest
141}
142
143// recordCandidate durably journals a freshly created candidate before it is
144// started. A candidate whose ID cannot be recorded must not keep running:
145// recovery could never find it, so it is removed and the failure surfaced.
146func (s *Service) recordCandidate(ctx context.Context, app, service, containerID string, journal candidateJournal) error {
147 if journal == nil {
148 return nil
149 }
150 if err := journal(ctx, containerID); err != nil {
151 retired := s.retireContainer(ctx, app, retireOptions{Service: service, Force: true}, containerID)
152 if !retired.Gone {
153 return fmt.Errorf("%w (candidate cleanup: %s)", err, cleanupDetail(retired))
154 }
155 return err
156 }
157 return nil
158}
159
160func (s *Service) createContainer(ctx context.Context, service string, config *domain.ContainerConfig) (*domain.Container, error) {
161 created, err := s.deps.Runtime.CreateContainer(ctx, config)
162 if err == nil {
163 return created, nil
164 }
165 // The engine-unsupported sentinel survives redaction so callers can
166 // map it to the structured runtime-unsupported envelope. It carries
167 // no host inventory (engine family/version only), and the device
168 // gate runs only for device-bearing creates, so this branch also
169 // covers services that declare both binds and devices.
170 if errors.Is(err, domain.ErrRuntimeUnsupported) {
171 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: %w", service, domain.ErrRuntimeUnsupported)
172 }
173 if len(config.Binds) > 0 {
174 // A CreateContainer failure is a runtime error, not a bind policy
175 // violation: policy was already enforced by resolveServiceBinds
176 // before this call. Runtime errors may embed the resolved host
177 // path, so redact the whole cause instead of mislabelling it and
178 // keep it out of operation journals and API/CLI responses.
179 return nil, fmt.Errorf("deployment: create container for service %q with administrative mounts: runtime error redacted", service)
180 }
181 if len(config.CDIDevices) > 0 {
182 // Same redaction rule for device-bearing creates: runtime errors
183 // may embed resolved CDI IDs, which are host inventory. Policy
184 // was already enforced by resolveServiceDevices before this call.
185 return nil, fmt.Errorf("deployment: create container for service %q with administrative devices: runtime error redacted", service)
186 }
187 return nil, fmt.Errorf("deployment: create container: %w", err)
188}
189
190// reserveVolumeOwnership durably records one app-owned volume as
191// attached BEFORE the runtime volume is created, so a crash between the
192// two leaves a protected record instead of an unowned volume. The
193// caller passes the validated incarnation ID from ensureIncarnationID,
194// which already assigned it, so no reload is needed for the labels.
195func (s *Service) reserveVolumeOwnership(ctx context.Context, app, appID, service, volumeName, runtimeName string) error {
196 ownership, err := s.deps.State.LoadOwnership(ctx, app)
197 if err != nil {
198 return fmt.Errorf("deployment: load ownership: %w", err)
199 }
200 if ownership.App == "" {
201 ownership.App = app
202 }
203 if ownership.ID == "" {
204 ownership.ID = appID
205 }
206 entry := domain.AppOwnedVolume{
207 Name: volumeName,
208 Service: service,
209 RuntimeName: runtimeName,
210 State: domain.AppResourceAttached,
211 }
212 replaced := false
213 for i, existing := range ownership.Volumes {
214 if existing.RuntimeName == runtimeName {
215 ownership.Volumes[i] = entry
216 replaced = true
217 break
218 }
219 }
220 if !replaced {
221 ownership.Volumes = append(ownership.Volumes, entry)
222 }
223 if err := s.deps.State.SaveOwnership(ctx, ownership); err != nil {
224 return fmt.Errorf("deployment: reserve volume ownership: %w", err)
225 }
226 return nil
227}
228
229// volumeProvenanceLabels is the label set stamped on an app-created
230// volume. Ownership is never inferred from the volume name, so the
231// labels must carry the full provenance: app, incarnation UUID, and
232// service. The managed marker is stamped here as well as merged by the
233// adapter, so provenance never depends on adapter behaviour alone.
234func volumeProvenanceLabels(app, appID, service, revision string) map[string]string {
235 labels := map[string]string{
236 domain.LabelManaged: "true",
237 domain.LabelApp: app,
238 domain.LabelAppService: service,
239 domain.LabelAppRevision: revision,
240 }
241 if appID != "" {
242 labels[domain.LabelAppID] = appID
243 }
244 return labels
245}
246
247// appLabels stamps engine ownership on every created container.
248func appLabels(app, service, revision string) map[string]string {
249 return map[string]string{
250 domain.LabelManaged: "true",
251 domain.LabelApp: app,
252 domain.LabelAppService: service,
253 domain.LabelAppRevision: revision,
254 }
255}
256
257// shortOp keeps container names short but unique per op.
258func shortOp(opID string) string {
259 trimmed := strings.TrimPrefix(opID, "op-")
260 if len(trimmed) > 12 {
261 trimmed = trimmed[:12]
262 }
263 if trimmed == "" {
264 return "op"
265 }
266 return trimmed
267}
268
269// serviceEnv resolves the service environment: the captured revision's
270// app-wide public env on every service, plus that service's own secret
271// values (memory only, UUID-keyed paths). Keys are emitted sorted and
272// deterministic. Overlapping public/secret keys are invalid with no
273// precedence; preflight rejects them before any mutation.
274func (s *Service) serviceEnv(ctx context.Context, app string, p pinnedService) ([]string, error) {
275 spec := p.spec
276 envMap := make(map[string]string, len(p.appEnv)+len(spec.Secrets))
277 for key, value := range p.appEnv {
278 envMap[key] = value
279 }
280 keys := make([]string, 0, len(spec.Secrets))
281 for envKey := range spec.Secrets {
282 keys = append(keys, envKey)
283 }
284 sort.Strings(keys)
285 if len(keys) == 0 {
286 return sortedEnv(envMap), nil
287 }
288 id, err := s.appSecretID(ctx, app)
289 if err != nil {
290 return nil, err
291 }
292 for _, envKey := range keys {
293 if _, overlap := envMap[envKey]; overlap {
294 return nil, fmt.Errorf("deployment: env key %q collides with a secret key in service %q: %w", envKey, spec.Name, domain.ErrInvalidAppSpec)
295 }
296 path := domain.AppSecretPathForID(id, app, spec.Name, spec.Secrets[envKey])
297 value, err := s.deps.Secrets.GetSecret(ctx, path)
298 if err != nil {
299 return nil, fmt.Errorf("deployment: secret %s (env %s) missing: %w", path, envKey, domain.ErrAppSecretMissing)
300 }
301 envMap[envKey] = value
302 }
303 return sortedEnv(envMap), nil
304}
305
306// sortedEnv renders a KEY=value list in sorted key order. Values stay in
307// memory for container creation only; nothing is persisted or logged.
308func sortedEnv(envMap map[string]string) []string {
309 if len(envMap) == 0 {
310 return nil
311 }
312 keys := make([]string, 0, len(envMap))
313 for key := range envMap {
314 keys = append(keys, key)
315 }
316 sort.Strings(keys)
317 env := make([]string, 0, len(keys))
318 for _, key := range keys {
319 env = append(env, key+"="+envMap[key])
320 }
321 return env
322}
323
324// pullImage fetches the pinned image from the installation registry.
325// External refs pull anonymously; installation-registry refs use the
326// internal credentials. A missing registry config disables pulling
327// (tests and pre-pulled environments).
328func (s *Service) pullImage(ctx context.Context, image string) (string, error) {
329 if err := s.validateImageSource(image, ""); err != nil {
330 return "", err
331 }
332 if s.deps.Registry.Domain == "" {
333 return image, nil
334 }
335 if !s.deps.ImagePolicy.IsInstallationImage(image) {
336 if err := s.deps.Runtime.PullImage(ctx, image); err != nil {
337 return "", fmt.Errorf("deployment: pull image %q: %w", image, err)
338 }
339 return image, nil
340 }
341 _, remainder, ok := strings.Cut(image, "/")
342 if !ok || !strings.Contains(remainder, "@sha256:") {
343 return "", fmt.Errorf("deployment: installation image must include an exact sha256 digest: %w", domain.ErrAppImageNotAllowed)
344 }
345 pullImage := s.deps.Registry.PullAddress + "/" + remainder
346 request := domain.ImagePullRequest{Reference: pullImage, Username: s.deps.Registry.Username, Password: s.deps.Registry.Password, Transport: domain.ImagePullTransportHTTP}
347 if err := s.deps.Runtime.PullImageWithOptions(ctx, request); err != nil {
348 return "", fmt.Errorf("deployment: pull installation image %q via configured local HTTP transport %q: %w", image, s.deps.Registry.PullAddress, err)
349 }
350 digest := remainder[strings.LastIndex(remainder, "@")+1:]
351 if err := s.deps.Runtime.VerifyImageDigest(ctx, pullImage, digest); err != nil {
352 return "", fmt.Errorf("deployment: verify pulled installation image %q: %w", image, err)
353 }
354 return pullImage, nil
355}
356
357// stripImageTag removes any tag or digest suffix from an image reference.
358// colon is a registry port (localhost:15500/e2e/web:v1 strips to
359// localhost:15500/e2e/web).
360func stripImageTag(ref string) string {
361 if base, _, ok := strings.Cut(ref, "@"); ok {
362 return base
363 }
364 lastSlash := strings.LastIndex(ref, "/")
365 if idx := strings.LastIndex(ref, ":"); idx > lastSlash {
366 return ref[:idx]
367 }
368 return ref
369}