Merge pull request 'feat(deployment): replace stateless services without downtime' (#21) from feat/zero-downtime-stateless-deploy into main

commit: b164fe9caae3310ed8925f9a3110b119361faed8
author: bnema · 5d ago · 2026-10-16 21:10 UTC
merge of: ccc19f7 + 253c38f · diff against the first parent
signature: SSH UnverifiedSHA256:VOCA5lYARHcbKDZ7GhTV45VmcjWnJ6ZQIBazj17LivA The key is unknown
refs: tag: v3.3.0

35 files · +2198 -199

M README.md +1 -1


@@ -154,7 +154,7 @@ gordon apps deploy blog --remote prod
154	154		- Private Docker/Podman registry on your VPS
155	155		- Declarative apps: one TOML file per app, explicit apply then deploy
156	156		- Domain-to-container routing through a smart TCP edge reverse proxy
157		-	- Sequential service replacement (withdraw traffic, replace one generation, verify readiness, republish)
	157	+	- Zero-downtime replacement of stateless TCP/HTTP services (start, verify readiness, switch traffic, stop the old one); services with volumes, binds, devices, or UDP interfaces are replaced sequentially
158	158		- Remote CLI management (daemon is the sole writer)
159	159		- Declarative per-service volumes, retained across lifecycle operations
160	160		- Per-service secrets in pass, app-wide public env in the manifest


@@ -154,7 +154,7 @@ gordon apps deploy blog --remote prod
154		- Private Docker/Podman registry on your VPS	154		- Private Docker/Podman registry on your VPS
155		- Declarative apps: one TOML file per app, explicit apply then deploy	155		- Declarative apps: one TOML file per app, explicit apply then deploy
156		- Domain-to-container routing through a smart TCP edge reverse proxy	156		- Domain-to-container routing through a smart TCP edge reverse proxy
157	-	- Sequential service replacement (withdraw traffic, replace one generation, verify readiness, republish)	157	+	- Zero-downtime replacement of stateless TCP/HTTP services (start, verify readiness, switch traffic, stop the old one); services with volumes, binds, devices, or UDP interfaces are replaced sequentially
158		- Remote CLI management (daemon is the sole writer)	158		- Remote CLI management (daemon is the sole writer)
159		- Declarative per-service volumes, retained across lifecycle operations	159		- Declarative per-service volumes, retained across lifecycle operations
160		- Per-service secrets in pass, app-wide public env in the manifest	160		- Per-service secrets in pass, app-wide public env in the manifest

M docs/concepts.md +15 -5


@@ -71,19 +71,29 @@ DATABASE_URL = "database-url"
71	71
72	72		## Updates
73	73
74		-	Gordon deploys an app's services one at a time in sorted order. A deploy may cause a short service interruption; there is no zero-downtime promise for app services. Gordon never runs two Gordon-managed generations of the same service at the same time.
	74	+	Gordon deploys an app's services one at a time in sorted order. A stateless TCP or HTTP service (no volume, bind, device, or UDP interface) is replaced without downtime; every other service is replaced sequentially and may be briefly interrupted. Gordon never runs two Gordon-managed generations of a service that owns a volume, a bind, or a device, and a UDP service is excluded because UDP sessions stay pinned to the backend they started on.
75	75
76		-	Every replaced service follows the same flow: preflight completes before anything is disrupted (image resolution and pull, secrets, volumes, networks, bind policy, reservations); the service is withdrawn from traffic; the old container is stopped and removed; the new container is created and started; the declared readiness probe runs; the new `ACTIVE` state is published; and traffic is rebuilt and published for the new container.
	76	+	Preflight completes before anything is disrupted (image resolution and pull, secrets, volumes, networks, bind policy, reservations). If preflight fails, the running service is left untouched.
	77	+
	78	+	A stateless service (no volume, bind, device, or UDP interface) is replaced with a short overlap:
	79	+
	80	+	```
	81	+	Preflight ─► start new ─► readiness ─► publish ACTIVE ─► route traffic ─► SIGTERM old ─► stop_grace ─► kill + remove old
	82	+	```
	83	+
	84	+	The old container keeps serving and stays routed until the new one is ready and traffic points at it. Only then does the old container receive SIGTERM, with its own `stop_grace` (default 30s) before it is killed. While both run, they share the service's network alias, so a request from another service of the app may reach either generation. A database migration must therefore stay backward compatible with the previous version of the service. If the new container never becomes ready, it is removed and the old one is not touched. If the old container cannot be removed after the switch, the deploy still succeeds and reports a cleanup warning. Before the switch Gordon durably marks the old container for retirement, so the periodic recovery pass keeps retrying it, with the old container's `stop_grace`, once the new container is verified; later deploys do not lose the cleanup.
	85	+
	86	+	A service with a volume, a bind, a device, or a UDP interface is replaced sequentially:
77	87
78	88		```
79	89		Preflight ─► withdraw traffic ─► stop + remove old ─► start new ─► readiness ─► publish ACTIVE ─► route traffic
80	90		```
81	91
82		-	If preflight fails, the running service is left untouched. If replacement fails after the old container was removed, the failure is explicit: Gordon does not recreate the old container during that operation and does not promise automatic data rollback. A stateful service's recovery inhibition stays in place until its never-published candidate is confirmed removed; a later boot/start/restart then clears it and rebuilds the generation `ACTIVE` records from its pinned digest, while a candidate that cannot be removed fails recovery closed.
	92	+	If replacement fails after the old container was removed, the failure is explicit: Gordon does not recreate the old container during that operation and does not promise automatic data rollback. A stateful service's recovery inhibition stays in place until its never-published candidate is confirmed removed; a later boot/start/restart then clears it and rebuilds the generation `ACTIVE` records from its pinned digest, while a candidate that cannot be removed fails recovery closed.
83	93
84		-	Deployment stops at the first service failure: services already deployed in that run are kept and are not rolled back. This flow covers every service, including TCP, UDP, mixed, and volume-owning services. An open UDP socket is not application readiness, and Gordon never restarts an old volume-owning image automatically after a replacement may have written data.
	94	+	Deployment stops at the first service failure: services already deployed in that run are kept and are not rolled back. This covers TCP, UDP, mixed, and volume-owning services. An open UDP socket is not application readiness, and Gordon never restarts an old volume-owning image automatically after a replacement may have written data.
85	95
86		-	A deploy interrupted after the replacement container was created but before it was published leaves the failure explicit: the candidate is recorded in the deployment journal, and Gordon removes it before it creates or rebuilds any generation of that app — at boot and before every mutation — so two generations of one service never run together. An interrupted operation is finalized instead of staying in flight: as a failure, or as the success it was when every step had already completed. If the recorded candidate cannot be removed, reconciliation fails closed: the operation is not finalized, stays the latest journal, and every later mutation is refused so a newer operation can never mask the orphan. The next recovery pass republishes routing within its 15-second cadence if the final traffic publication was the step that failed.
	96	+	A deploy interrupted after the replacement container was created but before it was published leaves the failure explicit: the candidate is recorded in the deployment journal, and Gordon removes it before it creates or rebuilds any generation of that app — at boot and before every mutation. For a stateless service the old container keeps serving and is kept. A deploy interrupted after the switch but before the old container was stopped leaves two containers running; the retirement mark makes the periodic recovery pass stop the old one, but only once that same pass has verified and routed the published container. `gordon apps stop` and `gordon apps remove` also stop any such old container, and fail, keeping the mark, if they cannot. Boot recovery never routes a persisted port it has not verified. An interrupted operation is finalized instead of staying in flight: as a failure, or as the success it was when every step had already completed. If a recorded candidate cannot be removed, reconciliation fails closed: the operation is not finalized, stays the latest journal, and every later mutation is refused so a newer operation can never mask the orphan. The next recovery pass republishes routing within its 15-second cadence if the final traffic publication was the step that failed.
87	97
88	98		Deploy is the only command that applies changes: it recreates a service whose image digest, spec, app environment, or secret values changed, and keeps an unchanged one, reported `unchanged`.
89	99

 ## Updates
-Gordon deploys an app's services one at a time in sorted order. A deploy may cause a short service interruption; there is no zero-downtime promise for app services. Gordon never runs two Gordon-managed generations of the same service at the same time.
+Gordon deploys an app's services one at a time in sorted order. A stateless TCP or HTTP service (no volume, bind, device, or UDP interface) is replaced without downtime; every other service is replaced sequentially and may be briefly interrupted. Gordon never runs two Gordon-managed generations of a service that owns a volume, a bind, or a device, and a UDP service is excluded because UDP sessions stay pinned to the backend they started on.
-Every replaced service follows the same flow: preflight completes before anything is disrupted (image resolution and pull, secrets, volumes, networks, bind policy, reservations); the service is withdrawn from traffic; the old container is stopped and removed; the new container is created and started; the declared readiness probe runs; the new `ACTIVE` state is published; and traffic is rebuilt and published for the new container.
+Preflight completes before anything is disrupted (image resolution and pull, secrets, volumes, networks, bind policy, reservations). If preflight fails, the running service is left untouched.
+A stateless service (no volume, bind, device, or UDP interface) is replaced with a short overlap:
+```
+Preflight ─► start new ─► readiness ─► publish ACTIVE ─► route traffic ─► SIGTERM old ─► stop_grace ─► kill + remove old
+```
+The old container keeps serving and stays routed until the new one is ready and traffic points at it. Only then does the old container receive SIGTERM, with its own `stop_grace` (default 30s) before it is killed. While both run, they share the service's network alias, so a request from another service of the app may reach either generation. A database migration must therefore stay backward compatible with the previous version of the service. If the new container never becomes ready, it is removed and the old one is not touched. If the old container cannot be removed after the switch, the deploy still succeeds and reports a cleanup warning. Before the switch Gordon durably marks the old container for retirement, so the periodic recovery pass keeps retrying it, with the old container's `stop_grace`, once the new container is verified; later deploys do not lose the cleanup.
+A service with a volume, a bind, a device, or a UDP interface is replaced sequentially:
 ```
 Preflight ─► withdraw traffic ─► stop + remove old ─► start new ─► readiness ─► publish ACTIVE ─► route traffic
 ```
-If preflight fails, the running service is left untouched. If replacement fails after the old container was removed, the failure is explicit: Gordon does not recreate the old container during that operation and does not promise automatic data rollback. A stateful service's recovery inhibition stays in place until its never-published candidate is confirmed removed; a later boot/start/restart then clears it and rebuilds the generation `ACTIVE` records from its pinned digest, while a candidate that cannot be removed fails recovery closed.
+If replacement fails after the old container was removed, the failure is explicit: Gordon does not recreate the old container during that operation and does not promise automatic data rollback. A stateful service's recovery inhibition stays in place until its never-published candidate is confirmed removed; a later boot/start/restart then clears it and rebuilds the generation `ACTIVE` records from its pinned digest, while a candidate that cannot be removed fails recovery closed.
-Deployment stops at the first service failure: services already deployed in that run are kept and are not rolled back. This flow covers every service, including TCP, UDP, mixed, and volume-owning services. An open UDP socket is not application readiness, and Gordon never restarts an old volume-owning image automatically after a replacement may have written data.
+Deployment stops at the first service failure: services already deployed in that run are kept and are not rolled back. This covers TCP, UDP, mixed, and volume-owning services. An open UDP socket is not application readiness, and Gordon never restarts an old volume-owning image automatically after a replacement may have written data.
-A deploy interrupted after the replacement container was created but before it was published leaves the failure explicit: the candidate is recorded in the deployment journal, and Gordon removes it before it creates or rebuilds any generation of that app — at boot and before every mutation — so two generations of one service never run together. An interrupted operation is finalized instead of staying in flight: as a failure, or as the success it was when every step had already completed. If the recorded candidate cannot be removed, reconciliation fails closed: the operation is not finalized, stays the latest journal, and every later mutation is refused so a newer operation can never mask the orphan. The next recovery pass republishes routing within its 15-second cadence if the final traffic publication was the step that failed.
+A deploy interrupted after the replacement container was created but before it was published leaves the failure explicit: the candidate is recorded in the deployment journal, and Gordon removes it before it creates or rebuilds any generation of that app — at boot and before every mutation. For a stateless service the old container keeps serving and is kept. A deploy interrupted after the switch but before the old container was stopped leaves two containers running; the retirement mark makes the periodic recovery pass stop the old one, but only once that same pass has verified and routed the published container. `gordon apps stop` and `gordon apps remove` also stop any such old container, and fail, keeping the mark, if they cannot. Boot recovery never routes a persisted port it has not verified. An interrupted operation is finalized instead of staying in flight: as a failure, or as the success it was when every step had already completed. If a recorded candidate cannot be removed, reconciliation fails closed: the operation is not finalized, stays the latest journal, and every later mutation is refused so a newer operation can never mask the orphan. The next recovery pass republishes routing within its 15-second cadence if the final traffic publication was the step that failed.
 Deploy is the only command that applies changes: it recreates a service whose image digest, spec, app environment, or secret values changed, and keeps an unchanged one, reported `unchanged`.

M docs/config/apps.md +14 -2


@@ -110,9 +110,17 @@ RCON is ordinary TCP: use TCP interfaces, no special RCON kind.
110	110
111	111		Image registry names and digest syntax are validated during manifest apply, resolution, deployment preflight, and immediately before every pull, including boot, restart, and recovery. Docker Hub (`docker.io` and canonical pull host `registry-1.docker.io`), `ghcr.io`, `quay.io`, and Gordon's configured registry are allowed by default. Add other exact registry hostname+port entries with `images.allowed_registries`; this does not configure credentials, and external resolution and pulls use anonymous access. When `images.require_digest` is enabled, every image reference, including Gordon registry images, must use `@sha256:<64 hex chars>`. See [Images](./images.md). This is a hostname allowlist, not DNS/IP validation or runtime egress enforcement.
112	112
	113	+	## Replacement on Deploy
	114	+
	115	+	A deploy recreates a service whose image digest, spec, environment, or secret values changed and keeps an unchanged one.
	116	+
	117	+	- A stateless TCP or HTTP service (no volume, bind, device, or UDP interface) is replaced without downtime. The new container starts and passes its readiness probe while the old one keeps serving; traffic then switches to the new container; only afterwards does the old container receive SIGTERM, and it is killed once its `stop_grace` (default `30s`) runs out. If the new container is not ready, it is removed and the old one is left untouched.
	118	+	- During the overlap both containers answer to the same network alias, so other services of the app may reach either generation for a short time. Database migrations must be backward compatible with the previous version of the service.
	119	+	- A service with a volume, bind, device, or UDP interface, or whose currently running generation has one, is replaced sequentially: traffic is withdrawn, the old container is stopped and removed, then the new one starts. It may be briefly interrupted, and two generations of it never run at once. UDP services are excluded from the overlap because UDP sessions stay pinned to the backend they started on.
	120	+
113	121		## Environment and Secrets
114	122
115		-	- `[env]` is app-wide public env injected into all services. Each key must be disjoint from every `[services.<name>.secrets]` key in the app.
	123	+	- `[env]` is app-wide public env injected into all services. Each key must be disjoint from every `[services.<name>.secrets]` key in the app. A change to `[env]` changes every service, so the next deploy recreates all of them, databases included. Prefer `[services.<name>.env]` for values only one service needs.
116	124		- `[services.<name>.env]` is public env for that service only. A key here overrides the same key in `[env]` for this service. Each key must be disjoint from the same service's `[services.<name>.secrets]` keys. Values follow the `[env]` rules: no newlines, 1 byte to 64 KiB, no secret references. Values are stored in revisions and shown in plain text, so use secrets for anything sensitive. A change shows as `service/<name>/env` in diffs and applies on the next deploy.
117	125		- `[services.<name>.secrets]` maps ENV var name to service-local secret name. Values are written with `gordon apps secrets set` and stay in pass under `gordon/apps/<uuid>/<service>/<name>`. Running containers keep the values they were created with; `gordon apps deploy` applies new values. `restart` does not.
118	126		- A secret entry is either a secret name string (env delivery) or a table `{ name = "...", as = "env" \| "file" }`. `as` defaults to `env`; any other key or value is rejected.

@@ -127,7 +135,7 @@ Image registry names and digest syntax are validated during manifest apply, reso
127	135		- Named volumes are Gordon-owned app data: created, labeled, retained, backed up, and pruned by Gordon. Administrative binds are operator-owned host locations: Gordon mounts them and never creates, deletes, owns, backs up, or prunes them.
128	136		- `[[services.<name>.bind]]` requires an absolute, normalized container `path` and an optional `readonly`. The policy's `read_only` and the bind's `readonly` force read-only together: either side wins and a manifest can never weaken its policy. Reserved destinations (`/`, `/proc`, `/sys`, `/dev`, `/boot` and their children) and paths colliding with a declared volume or another bind are rejected at apply time.
129	137		- `devices` lists logical device names granted by `[app_devices.<name>]` policy the operator declares in `gordon.toml`; a device whose name has no matching policy, or whose policy does not allow the app+service pair, is rejected at apply time. Gordon resolves each name to explicit CDI IDs at activation time and encodes them as one native CDI `DeviceRequest`. Revisions persist the logical names, never the host resolution. A device add/remove shows as `service/<name>/devices` in diffs; reorder-only input is a no-op. Changing a mapping never recreates a running container: the next deploy serves the new resolution. Revoking a grant fails subsequent deploys closed while the running service is untouched. Device-bearing creates require Podman 5.4+ or Docker 28.3+ with native CDI configured; older or unrecognized engines return a structured `runtime-unsupported` error and never run without devices. Gordon installs no drivers, manages no quotas, and injects no `NVIDIA_*` environment: images carry their own runtime expectations.
130		-	- A service with any bind runs as a single writer, like a volume-backed service: replacements never serve two generations at once. `gordon apps status` and `apps show` mark such services `single-writer` (`restart_unsafe` in JSON); the mark is informational. See [apps status](../cli/apps.md#gordon-apps-status).
	138	+	- A service with any bind runs as a single writer, like a volume-backed service: replacements never run two generations at once. `gordon apps status` and `apps show` mark such services `single-writer` (`restart_unsafe` in JSON); the mark is informational. See [apps status](../cli/apps.md#gordon-apps-status).
131	139		- A volume declared `readonly = true` is mounted read-only in the container; the service cannot modify protected data.
132	140		- `[[services.<name>.database]]` declares databases explicitly (no image inference). Only PostgreSQL is supported, and each database declares its own backup `schedule` (`hourly`, `daily`, `weekly`, or `monthly`).
133	141		- `[services.<name>.backup]` lists the declared databases (`postgres`) and volumes (`volume`) that are backup targets. A declared database or volume that is not referenced here is never backed up. Schedules follow the declaration through deploys; stored backups are never deleted when declarations change.

@@ -158,6 +166,10 @@ A container port declared by both an internal HTTP interface and an externally b
158	166
159	167		`type = "http"` requires an origin-form `path` beginning with a single `/`. Absolute URLs, authority forms such as `@host:port`, scheme-relative paths, and control characters are rejected at apply time. The probe always dials the declared loopback backend, never follows redirects, ignores environment proxy settings, and is bounded per request and for the whole operation. A public or otherwise published interface keeps this loopback probe. An internal HTTP port is instead probed over the app private network by one bounded, short-lived helper, using HTTP or TCP according to the interface's `[services.<name>.readiness]` type; the internal port is never temporarily published to the host to probe it.
160	168
	169	+	`type = "tcp"` on a published port is ready when a connection to the loopback backend is accepted and then either receives data or stays open for 300ms. A connection closed right after accept counts as not ready: the host port relay of rootless Docker accepts on behalf of a container that does not listen yet. A workload that accepts and closes immediately by design should use `log` readiness instead.
	170	+
	171	+	When a service declares no readiness (or `none`), deploy and restart still probe it in two cases: a stateless service whose interfaces are all public HTTP gets an `http` probe on `/`, and a stateless service with exactly one TCP interface and no HTTP or UDP interface gets a `tcp` probe on that port. Other services without a declared probe are not checked.
	172	+
161	173		## TLS
162	174
163	175		TLS applies to public interfaces only; an internal interface never declares `tls`.

 Image registry names and digest syntax are validated during manifest apply, resolution, deployment preflight, and immediately before every pull, including boot, restart, and recovery. Docker Hub (`docker.io` and canonical pull host `registry-1.docker.io`), `ghcr.io`, `quay.io`, and Gordon's configured registry are allowed by default. Add other exact registry hostname+port entries with `images.allowed_registries`; this does not configure credentials, and external resolution and pulls use anonymous access. When `images.require_digest` is enabled, every image reference, including Gordon registry images, must use `@sha256:<64 hex chars>`. See [Images](./images.md). This is a hostname allowlist, not DNS/IP validation or runtime egress enforcement.
+## Replacement on Deploy
+A deploy recreates a service whose image digest, spec, environment, or secret values changed and keeps an unchanged one.
+- A stateless TCP or HTTP service (no volume, bind, device, or UDP interface) is replaced without downtime. The new container starts and passes its readiness probe while the old one keeps serving; traffic then switches to the new container; only afterwards does the old container receive SIGTERM, and it is killed once its `stop_grace` (default `30s`) runs out. If the new container is not ready, it is removed and the old one is left untouched.
+- During the overlap both containers answer to the same network alias, so other services of the app may reach either generation for a short time. Database migrations must be backward compatible with the previous version of the service.
+- A service with a volume, bind, device, or UDP interface, or whose currently running generation has one, is replaced sequentially: traffic is withdrawn, the old container is stopped and removed, then the new one starts. It may be briefly interrupted, and two generations of it never run at once. UDP services are excluded from the overlap because UDP sessions stay pinned to the backend they started on.
 ## Environment and Secrets
-- `[env]` is app-wide public env injected into all services. Each key must be disjoint from every `[services.<name>.secrets]` key in the app.
+- `[env]` is app-wide public env injected into all services. Each key must be disjoint from every `[services.<name>.secrets]` key in the app. A change to `[env]` changes every service, so the next deploy recreates all of them, databases included. Prefer `[services.<name>.env]` for values only one service needs.
 - `[services.<name>.env]` is public env for that service only. A key here overrides the same key in `[env]` for this service. Each key must be disjoint from the same service's `[services.<name>.secrets]` keys. Values follow the `[env]` rules: no newlines, 1 byte to 64 KiB, no secret references. Values are stored in revisions and shown in plain text, so use secrets for anything sensitive. A change shows as `service/<name>/env` in diffs and applies on the next deploy.
 - `[services.<name>.secrets]` maps ENV var name to service-local secret name. Values are written with `gordon apps secrets set` and stay in pass under `gordon/apps/<uuid>/<service>/<name>`. Running containers keep the values they were created with; `gordon apps deploy` applies new values. `restart` does not.
 - A secret entry is either a secret name string (env delivery) or a table `{ name = "...", as = "env" | "file" }`. `as` defaults to `env`; any other key or value is rejected.
 - Named volumes are Gordon-owned app data: created, labeled, retained, backed up, and pruned by Gordon. Administrative binds are operator-owned host locations: Gordon mounts them and never creates, deletes, owns, backs up, or prunes them.
 - `[[services.<name>.bind]]` requires an absolute, normalized container `path` and an optional `readonly`. The policy's `read_only` and the bind's `readonly` force read-only together: either side wins and a manifest can never weaken its policy. Reserved destinations (`/`, `/proc`, `/sys`, `/dev`, `/boot` and their children) and paths colliding with a declared volume or another bind are rejected at apply time.
 - `devices` lists logical device names granted by `[app_devices.<name>]` policy the operator declares in `gordon.toml`; a device whose name has no matching policy, or whose policy does not allow the app+service pair, is rejected at apply time. Gordon resolves each name to explicit CDI IDs at activation time and encodes them as one native CDI `DeviceRequest`. Revisions persist the logical names, never the host resolution. A device add/remove shows as `service/<name>/devices` in diffs; reorder-only input is a no-op. Changing a mapping never recreates a running container: the next deploy serves the new resolution. Revoking a grant fails subsequent deploys closed while the running service is untouched. Device-bearing creates require Podman 5.4+ or Docker 28.3+ with native CDI configured; older or unrecognized engines return a structured `runtime-unsupported` error and never run without devices. Gordon installs no drivers, manages no quotas, and injects no `NVIDIA_*` environment: images carry their own runtime expectations.
-- A service with any bind runs as a single writer, like a volume-backed service: replacements never serve two generations at once. `gordon apps status` and `apps show` mark such services `single-writer` (`restart_unsafe` in JSON); the mark is informational. See [apps status](../cli/apps.md#gordon-apps-status).
+- A service with any bind runs as a single writer, like a volume-backed service: replacements never run two generations at once. `gordon apps status` and `apps show` mark such services `single-writer` (`restart_unsafe` in JSON); the mark is informational. See [apps status](../cli/apps.md#gordon-apps-status).
 - A volume declared `readonly = true` is mounted read-only in the container; the service cannot modify protected data.
 - `[[services.<name>.database]]` declares databases explicitly (no image inference). Only PostgreSQL is supported, and each database declares its own backup `schedule` (`hourly`, `daily`, `weekly`, or `monthly`).
 - `[services.<name>.backup]` lists the declared databases (`postgres`) and volumes (`volume`) that are backup targets. A declared database or volume that is not referenced here is never backed up. Schedules follow the declaration through deploys; stored backups are never deleted when declarations change.
 `type = "http"` requires an origin-form `path` beginning with a single `/`. Absolute URLs, authority forms such as `@host:port`, scheme-relative paths, and control characters are rejected at apply time. The probe always dials the declared loopback backend, never follows redirects, ignores environment proxy settings, and is bounded per request and for the whole operation. A public or otherwise published interface keeps this loopback probe. An internal HTTP port is instead probed over the app private network by one bounded, short-lived helper, using HTTP or TCP according to the interface's `[services.<name>.readiness]` type; the internal port is never temporarily published to the host to probe it.
+`type = "tcp"` on a published port is ready when a connection to the loopback backend is accepted and then either receives data or stays open for 300ms. A connection closed right after accept counts as not ready: the host port relay of rootless Docker accepts on behalf of a container that does not listen yet. A workload that accepts and closes immediately by design should use `log` readiness instead.
+When a service declares no readiness (or `none`), deploy and restart still probe it in two cases: a stateless service whose interfaces are all public HTTP gets an `http` probe on `/`, and a stateless service with exactly one TCP interface and no HTTP or UDP interface gets a `tcp` probe on that port. Other services without a declared probe are not checked.
 ## TLS
 TLS applies to public interfaces only; an internal interface never declares `tls`.

M docs/deployment/index.md +15 -3


@@ -146,15 +146,27 @@ docker push gordon.example.com/myapp:$VERSION
146	146
147	147		## Updates
148	148
149		-	Gordon deploys an app's services one at a time in sorted order. A deploy may cause a short service interruption; there is no zero-downtime promise for app services, and Gordon never runs two Gordon-managed generations of the same service at the same time.
	149	+	Gordon deploys an app's services one at a time in sorted order. A stateless TCP or HTTP service (no volume, bind, device, or UDP interface) is replaced without downtime; a service that owns a volume, bind, or device is replaced sequentially and may be briefly interrupted, because Gordon never runs two Gordon-managed generations of such a service at the same time. A UDP service is also replaced sequentially: UDP sessions stay pinned to the backend they started on, so they could not follow the switch.
	150	+
	151	+	Stateless service (no volume, bind, device, or UDP interface):
	152	+
	153	+	```
	154	+	Preflight ─► start new ─► readiness ─► publish ACTIVE ─► route traffic ─► SIGTERM old ─► stop_grace ─► kill + remove old
	155	+	```
	156	+
	157	+	Service with a volume, bind, device, or UDP interface:
150	158
151	159		```
152	160		Preflight ─► withdraw traffic ─► stop + remove old ─► start new ─► readiness ─► publish ACTIVE ─► route traffic
153	161		```
154	162
155		-	Preflight (image resolution and pull, secrets, volumes, networks, bind policy, reservations) completes before anything is disrupted, so a preflight failure leaves the running service untouched. If replacement fails after the old container was removed, the failure is explicit: Gordon does not recreate the old container during that operation and does not promise automatic data rollback. A stateful service's recovery inhibition stays in place until its never-published candidate is confirmed removed; a later boot/start/restart then clears it and rebuilds the generation `ACTIVE` records from its pinned digest, while a candidate that cannot be removed fails recovery closed. Deployment stops at the first service failure: services already deployed in that run are kept and are not rolled back.
	163	+	Preflight (image resolution and pull, secrets, volumes, networks, bind policy, reservations) completes before anything is disrupted, so a preflight failure leaves the running service untouched.
	164	+
	165	+	For a stateless service the old container serves until the new one is ready and routed, then receives SIGTERM and is killed after its `stop_grace` (default 30s). During the overlap both containers share the service's network alias, so other services of the app may reach either one. Database migrations must stay backward compatible with the previous version of the service. A new container that fails readiness is removed and the old one is not touched. An old container that cannot be removed is reported as a cleanup warning without failing the deploy; it stays durably marked for retirement, and the periodic recovery pass retries it with its own `stop_grace` once that pass has verified the new container, even after later deploys. A retirement is only complete once the old container's loopback port claims are released; until then it is retried. `gordon apps stop` and `gordon apps remove` also stop such an old container, and fail, keeping the mark, if they cannot.
	166	+
	167	+	For a sequential replacement, a failure after the old container was removed is explicit: Gordon does not recreate the old container during that operation and does not promise automatic data rollback. A stateful service's recovery inhibition stays in place until its never-published candidate is confirmed removed; a later boot/start/restart then clears it and rebuilds the generation `ACTIVE` records from its pinned digest, while a candidate that cannot be removed fails recovery closed. Deployment stops at the first service failure: services already deployed in that run are kept and are not rolled back.
156	168
157		-	If a deploy is interrupted between creating the replacement container and publishing it, that candidate is recorded durably and Gordon removes it before creating or rebuilding any generation of that app — at boot and before every mutation — so two generations of one service never run together. An interrupted operation is finalized instead of staying in flight: as a failure, or as the success it was when every step had already completed. If the recorded candidate cannot be removed, reconciliation fails closed: the operation is not finalized, stays the latest journal, and every later mutation is refused so a newer operation can never mask the orphan. A failed final traffic publication is republished by the next recovery pass, within 15 seconds by default.
	169	+	If a deploy is interrupted between creating the replacement container and publishing it, that candidate is recorded durably and Gordon removes it before creating or rebuilding any generation of that app — at boot and before every mutation; a stateless service's old container keeps serving. If it is interrupted after the switch but before the old container was stopped, the periodic recovery pass verifies and routes the published container and then stops the old one. An interrupted operation is finalized instead of staying in flight: as a failure, or as the success it was when every step had already completed. If a recorded candidate cannot be removed, reconciliation fails closed: the operation is not finalized, stays the latest journal, and every later mutation is refused so a newer operation can never mask the orphan. A failed final traffic publication is republished by the next recovery pass, within 15 seconds by default.
158	170
159	171		## Related
160	172

 ## Updates
-Gordon deploys an app's services one at a time in sorted order. A deploy may cause a short service interruption; there is no zero-downtime promise for app services, and Gordon never runs two Gordon-managed generations of the same service at the same time.
+Gordon deploys an app's services one at a time in sorted order. A stateless TCP or HTTP service (no volume, bind, device, or UDP interface) is replaced without downtime; a service that owns a volume, bind, or device is replaced sequentially and may be briefly interrupted, because Gordon never runs two Gordon-managed generations of such a service at the same time. A UDP service is also replaced sequentially: UDP sessions stay pinned to the backend they started on, so they could not follow the switch.
+Stateless service (no volume, bind, device, or UDP interface):
+```
+Preflight ─► start new ─► readiness ─► publish ACTIVE ─► route traffic ─► SIGTERM old ─► stop_grace ─► kill + remove old
+```
+Service with a volume, bind, device, or UDP interface:
 ```
 Preflight ─► withdraw traffic ─► stop + remove old ─► start new ─► readiness ─► publish ACTIVE ─► route traffic
 ```
-Preflight (image resolution and pull, secrets, volumes, networks, bind policy, reservations) completes before anything is disrupted, so a preflight failure leaves the running service untouched. If replacement fails after the old container was removed, the failure is explicit: Gordon does not recreate the old container during that operation and does not promise automatic data rollback. A stateful service's recovery inhibition stays in place until its never-published candidate is confirmed removed; a later boot/start/restart then clears it and rebuilds the generation `ACTIVE` records from its pinned digest, while a candidate that cannot be removed fails recovery closed. Deployment stops at the first service failure: services already deployed in that run are kept and are not rolled back.
+Preflight (image resolution and pull, secrets, volumes, networks, bind policy, reservations) completes before anything is disrupted, so a preflight failure leaves the running service untouched.
+For a stateless service the old container serves until the new one is ready and routed, then receives SIGTERM and is killed after its `stop_grace` (default 30s). During the overlap both containers share the service's network alias, so other services of the app may reach either one. Database migrations must stay backward compatible with the previous version of the service. A new container that fails readiness is removed and the old one is not touched. An old container that cannot be removed is reported as a cleanup warning without failing the deploy; it stays durably marked for retirement, and the periodic recovery pass retries it with its own `stop_grace` once that pass has verified the new container, even after later deploys. A retirement is only complete once the old container's loopback port claims are released; until then it is retried. `gordon apps stop` and `gordon apps remove` also stop such an old container, and fail, keeping the mark, if they cannot.
+For a sequential replacement, a failure after the old container was removed is explicit: Gordon does not recreate the old container during that operation and does not promise automatic data rollback. A stateful service's recovery inhibition stays in place until its never-published candidate is confirmed removed; a later boot/start/restart then clears it and rebuilds the generation `ACTIVE` records from its pinned digest, while a candidate that cannot be removed fails recovery closed. Deployment stops at the first service failure: services already deployed in that run are kept and are not rolled back.
-If a deploy is interrupted between creating the replacement container and publishing it, that candidate is recorded durably and Gordon removes it before creating or rebuilding any generation of that app — at boot and before every mutation — so two generations of one service never run together. An interrupted operation is finalized instead of staying in flight: as a failure, or as the success it was when every step had already completed. If the recorded candidate cannot be removed, reconciliation fails closed: the operation is not finalized, stays the latest journal, and every later mutation is refused so a newer operation can never mask the orphan. A failed final traffic publication is republished by the next recovery pass, within 15 seconds by default.
+If a deploy is interrupted between creating the replacement container and publishing it, that candidate is recorded durably and Gordon removes it before creating or rebuilding any generation of that app — at boot and before every mutation; a stateless service's old container keeps serving. If it is interrupted after the switch but before the old container was stopped, the periodic recovery pass verifies and routes the published container and then stops the old one. An interrupted operation is finalized instead of staying in flight: as a failure, or as the success it was when every step had already completed. If a recorded candidate cannot be removed, reconciliation fails closed: the operation is not finalized, stays the latest journal, and every later mutation is refused so a newer operation can never mask the orphan. A failed final traffic publication is republished by the next recovery pass, within 15 seconds by default.
 ## Related

M docs/deployment/rollback.md +1 -1


@@ -12,7 +12,7 @@ gordon apps apply --file blog.toml --remote prod
12	12		gordon apps deploy blog --remote prod
13	13		```
14	14
15		-	A rollback deploy replaces services one at a time like any other deploy: a short service interruption is possible, and Gordon never runs two generations of the same service at the same time. If the replacement fails after the old container was removed, Gordon does not recreate it and does not promise automatic data rollback.
	15	+	A rollback deploy replaces services one at a time like any other deploy: a stateless TCP or HTTP service is replaced without downtime, while a service with a volume, bind, device, or UDP interface is replaced sequentially, may be briefly interrupted, and never has two generations running at once. If a sequential replacement fails after the old container was removed, Gordon does not recreate it and does not promise automatic data rollback.
16	16
17	17		## Re-push the Previous Image as latest
18	18


@@ -12,7 +12,7 @@ gordon apps apply --file blog.toml --remote prod
12		gordon apps deploy blog --remote prod	12		gordon apps deploy blog --remote prod
13		```	13		```
14			14
15	-	A rollback deploy replaces services one at a time like any other deploy: a short service interruption is possible, and Gordon never runs two generations of the same service at the same time. If the replacement fails after the old container was removed, Gordon does not recreate it and does not promise automatic data rollback.	15	+	A rollback deploy replaces services one at a time like any other deploy: a stateless TCP or HTTP service is replaced without downtime, while a service with a volume, bind, device, or UDP interface is replaced sequentially, may be briefly interrupted, and never has two generations running at once. If a sequential replacement fails after the old container was removed, Gordon does not recreate it and does not promise automatic data rollback.
16			16
17		## Re-push the Previous Image as latest	17		## Re-push the Previous Image as latest
18			18

M docs/getting-started.md +1 -1


@@ -197,7 +197,7 @@ gordon images push myapp --build --remote prod
197	197		gordon apps deploy blog --remote prod
198	198		```
199	199
200		-	Gordon deploys services one at a time in sorted order. A deploy may cause a short service interruption, and Gordon never runs two Gordon-managed generations of the same service at the same time. If preflight fails, the running service is left untouched. This applies to every service, including TCP, UDP, and volume-owning services.
	200	+	Gordon deploys services one at a time in sorted order. A stateless TCP or HTTP service (no volume, bind, device, or UDP interface) is replaced without downtime: the new container starts and becomes ready while the old one keeps serving, then traffic switches and the old one is stopped. A service that owns a volume, bind, device, or UDP interface is replaced sequentially, may be briefly interrupted, and never has two generations running at once. If preflight fails, the running service is left untouched.
201	201
202	202		## Next Steps
203	203


@@ -197,7 +197,7 @@ gordon images push myapp --build --remote prod
197		gordon apps deploy blog --remote prod	197		gordon apps deploy blog --remote prod
198		```	198		```
199			199
200	-	Gordon deploys services one at a time in sorted order. A deploy may cause a short service interruption, and Gordon never runs two Gordon-managed generations of the same service at the same time. If preflight fails, the running service is left untouched. This applies to every service, including TCP, UDP, and volume-owning services.	200	+	Gordon deploys services one at a time in sorted order. A stateless TCP or HTTP service (no volume, bind, device, or UDP interface) is replaced without downtime: the new container starts and becomes ready while the old one keeps serving, then traffic switches and the old one is stopped. A service that owns a volume, bind, device, or UDP interface is replaced sequentially, may be briefly interrupted, and never has two generations running at once. If preflight fails, the running service is left untouched.
201			201
202		## Next Steps	202		## Next Steps
203			203

M docs/index.md +1 -1


@@ -10,7 +10,7 @@ Gordon runs on your VPS and provides:
10	10		- Declarative Apps - One TOML file per app, explicit apply then deploy
11	11		- HTTP Reverse Proxy - Routes app hosts to containers
12	12		- Push, Apply, Deploy - Push stores images; deploy is always explicit
13		-	- Sequential Service Replacement - One service generation at a time: withdraw traffic, replace, verify readiness, republish
	13	+	- Zero-Downtime Stateless Deploys - Stateless services start the new generation, verify readiness, switch traffic, then stop the old one; services with volumes, binds, devices, or UDP interfaces are replaced sequentially
14	14		- Single Binary - ~15MB RAM footprint
15	15
16	16		## How It Works


@@ -10,7 +10,7 @@ Gordon runs on your VPS and provides:
10		- Declarative Apps - One TOML file per app, explicit apply then deploy	10		- Declarative Apps - One TOML file per app, explicit apply then deploy
11		- HTTP Reverse Proxy - Routes app hosts to containers	11		- HTTP Reverse Proxy - Routes app hosts to containers
12		- Push, Apply, Deploy - Push stores images; deploy is always explicit	12		- Push, Apply, Deploy - Push stores images; deploy is always explicit
13	-	- Sequential Service Replacement - One service generation at a time: withdraw traffic, replace, verify readiness, republish	13	+	- Zero-Downtime Stateless Deploys - Stateless services start the new generation, verify readiness, switch traffic, then stop the old one; services with volumes, binds, devices, or UDP interfaces are replaced sequentially
14		- Single Binary - ~15MB RAM footprint	14		- Single Binary - ~15MB RAM footprint
15			15
16		## How It Works	16		## How It Works

M internal/domain/app_state.go +9 -2


@@ -305,6 +305,10 @@ type AppRecoveryInhibition struct {
305	305		Reason string `json:"reason"`
306	306		Operation string `json:"operation,omitempty"`
307	307		CreatedAt time.Time `json:"created_at"`
	308	+	// StopGrace is the effective stop grace of the generation to retire
	309	+	// (retirement-pending only). Zero, as in records written before it
	310	+	// existed, means the app default.
	311	+	StopGrace time.Duration `json:"stop_grace,omitempty"`
308	312		}
309	313
310	314		// AppRecoveryInhibition reasons.

@@ -312,8 +316,11 @@ const (
312	316		// AppInhibitReplacementPending marks the generation being replaced
313	317		// while a volume-owning candidate may already be writing.
314	318		AppInhibitReplacementPending = "replacement-pending"
315		-	// AppInhibitRetirementPending marks a published predecessor that must
316		-	// remain stopped and be retired by a later reconciliation pass.
	319	+	// AppInhibitRetirementPending marks a superseded generation that an
	320	+	// overlap replacement left running beside its published successor. The
	321	+	// periodic reconciliation retires it, with StopGrace, once the
	322	+	// successor is verified and routed; it is never retired, and the
	323	+	// marker is dropped, while ACTIVE names it.
317	324		AppInhibitRetirementPending = "retirement-pending"
318	325		)
319	326

 	Reason      string    `json:"reason"`
 	Operation   string    `json:"operation,omitempty"`
 	CreatedAt   time.Time `json:"created_at"`
+	// StopGrace is the effective stop grace of the generation to retire
+	// (retirement-pending only). Zero, as in records written before it
+	// existed, means the app default.
+	StopGrace time.Duration `json:"stop_grace,omitempty"`
 }
 // AppRecoveryInhibition reasons.
 	// AppInhibitReplacementPending marks the generation being replaced
 	// while a volume-owning candidate may already be writing.
 	AppInhibitReplacementPending = "replacement-pending"
-	// AppInhibitRetirementPending marks a published predecessor that must
+	// AppInhibitRetirementPending marks a superseded generation that an
-	// remain stopped and be retired by a later reconciliation pass.
+	// overlap replacement left running beside its published successor. The
+	// periodic reconciliation retires it, with StopGrace, once the
+	// successor is verified and routed; it is never retired, and the
+	// marker is dropped, while ACTIVE names it.
 	AppInhibitRetirementPending = "retirement-pending"
 )

M internal/usecase/deployment/bind_policy_internal_test.go +43 -2


@@ -227,7 +227,47 @@ func TestSingleWriterRequiredAndImplicitHTTPReadiness(t *testing.T) {
227	227		TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 5432, Publish: "5432"}},
228	228		}
229	229		assert.False(t, implicitHTTPProbe(l4Only), "an HTTP GET proves nothing about an L4 workload")
230		-	assert.Empty(t, deploymentReadiness(l4Only).Readiness.Type)
	230	+	})
	231	+
	232	+	t.Run("undeclared probe on a stateless single TCP service is the implicit TCP probe", func(t *testing.T) {
	233	+	for _, readiness := range []string{"", domain.AppReadinessNone} {
	234	+	spec := domain.AppService{
	235	+	Name: "relay",
	236	+	TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 9000, Publish: "9000"}},
	237	+	Readiness: domain.AppReadiness{Type: readiness},
	238	+	}
	239	+	assert.True(t, implicitTCPProbe(spec))
	240	+	assert.Equal(t, domain.AppReadinessTCP, deploymentReadiness(spec).Readiness.Type)
	241	+	}
	242	+	})
	243	+
	244	+	t.Run("no implicit TCP probe on stateful, multi-port, mixed, or UDP services", func(t *testing.T) {
	245	+	tcp := func() domain.AppService {
	246	+	return domain.AppService{
	247	+	Name: "db",
	248	+	TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 5432, Publish: "5432"}},
	249	+	}
	250	+	}
	251	+	withVolume := tcp()
	252	+	withVolume.Volumes = []domain.AppVolume{{Name: "data", Path: "/var/lib/postgresql/data"}}
	253	+	withBind := tcp()
	254	+	withBind.Binds = []domain.AppBind{{Name: "config", Path: "/etc/app.conf"}}
	255	+	twoPorts := tcp()
	256	+	twoPorts.TCP = append(twoPorts.TCP, domain.AppTCPInterface{Entrypoint: "tcp2", Port: 5433, Publish: "5433"})
	257	+	withUDP := tcp()
	258	+	withUDP.UDP = []domain.AppUDPInterface{{Entrypoint: "udp", Port: 5432}}
	259	+	mixed := tcp()
	260	+	mixed.HTTP = []domain.AppHTTPInterface{{Port: 8080, Visibility: domain.AppVisibilityInternal}}
	261	+	declaredLog := tcp()
	262	+	declaredLog.Readiness = domain.AppReadiness{Type: domain.AppReadinessLog}
	263	+
	264	+	for name, spec := range map[string]domain.AppService{
	265	+	"volume": withVolume, "bind": withBind, "two TCP ports": twoPorts,
	266	+	"UDP": withUDP, "HTTP interface": mixed, "declared log": declaredLog,
	267	+	} {
	268	+	assert.False(t, implicitTCPProbe(spec), name)
	269	+	assert.Equal(t, spec.Readiness.Type, deploymentReadiness(spec).Readiness.Type, name)
	270	+	}
231	271		})
232	272
233	273		t.Run("declared probes are never replaced by the implicit HTTP probe", func(t *testing.T) {

@@ -245,10 +285,11 @@ func TestSingleWriterRequiredAndImplicitHTTPReadiness(t *testing.T) {
245	285		assert.Equal(t, domain.AppReadinessHTTP, deploymentReadiness(spec).Readiness.Type)
246	286		})
247	287
248		-	t.Run("an explicit none on an L4 service stays unchecked", func(t *testing.T) {
	288	+	t.Run("an explicit none on a stateful L4 service stays unchecked", func(t *testing.T) {
249	289		spec := domain.AppService{
250	290		Name: "db",
251	291		TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 5432, Publish: "5432"}},
	292	+	Volumes: []domain.AppVolume{{Name: "data", Path: "/var/lib/postgresql/data"}},
252	293		Readiness: domain.AppReadiness{Type: domain.AppReadinessNone},
253	294		}
254	295		assert.Equal(t, domain.AppReadinessNone, deploymentReadiness(spec).Readiness.Type)


@@ -227,7 +227,47 @@ func TestSingleWriterRequiredAndImplicitHTTPReadiness(t *testing.T) {
227		TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 5432, Publish: "5432"}},	227		TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 5432, Publish: "5432"}},
228		}	228		}
229		assert.False(t, implicitHTTPProbe(l4Only), "an HTTP GET proves nothing about an L4 workload")	229		assert.False(t, implicitHTTPProbe(l4Only), "an HTTP GET proves nothing about an L4 workload")
230	-	assert.Empty(t, deploymentReadiness(l4Only).Readiness.Type)	230	+	})
			231	+
			232	+	t.Run("undeclared probe on a stateless single TCP service is the implicit TCP probe", func(t *testing.T) {
			233	+	for _, readiness := range []string{"", domain.AppReadinessNone} {
			234	+	spec := domain.AppService{
			235	+	Name: "relay",
			236	+	TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 9000, Publish: "9000"}},
			237	+	Readiness: domain.AppReadiness{Type: readiness},
			238	+	}
			239	+	assert.True(t, implicitTCPProbe(spec))
			240	+	assert.Equal(t, domain.AppReadinessTCP, deploymentReadiness(spec).Readiness.Type)
			241	+	}
			242	+	})
			243	+
			244	+	t.Run("no implicit TCP probe on stateful, multi-port, mixed, or UDP services", func(t *testing.T) {
			245	+	tcp := func() domain.AppService {
			246	+	return domain.AppService{
			247	+	Name: "db",
			248	+	TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 5432, Publish: "5432"}},
			249	+	}
			250	+	}
			251	+	withVolume := tcp()
			252	+	withVolume.Volumes = []domain.AppVolume{{Name: "data", Path: "/var/lib/postgresql/data"}}
			253	+	withBind := tcp()
			254	+	withBind.Binds = []domain.AppBind{{Name: "config", Path: "/etc/app.conf"}}
			255	+	twoPorts := tcp()
			256	+	twoPorts.TCP = append(twoPorts.TCP, domain.AppTCPInterface{Entrypoint: "tcp2", Port: 5433, Publish: "5433"})
			257	+	withUDP := tcp()
			258	+	withUDP.UDP = []domain.AppUDPInterface{{Entrypoint: "udp", Port: 5432}}
			259	+	mixed := tcp()
			260	+	mixed.HTTP = []domain.AppHTTPInterface{{Port: 8080, Visibility: domain.AppVisibilityInternal}}
			261	+	declaredLog := tcp()
			262	+	declaredLog.Readiness = domain.AppReadiness{Type: domain.AppReadinessLog}
			263	+
			264	+	for name, spec := range map[string]domain.AppService{
			265	+	"volume": withVolume, "bind": withBind, "two TCP ports": twoPorts,
			266	+	"UDP": withUDP, "HTTP interface": mixed, "declared log": declaredLog,
			267	+	} {
			268	+	assert.False(t, implicitTCPProbe(spec), name)
			269	+	assert.Equal(t, spec.Readiness.Type, deploymentReadiness(spec).Readiness.Type, name)
			270	+	}
231		})	271		})
232			272
233		t.Run("declared probes are never replaced by the implicit HTTP probe", func(t *testing.T) {	273		t.Run("declared probes are never replaced by the implicit HTTP probe", func(t *testing.T) {

@@ -245,10 +285,11 @@ func TestSingleWriterRequiredAndImplicitHTTPReadiness(t *testing.T) {
245		assert.Equal(t, domain.AppReadinessHTTP, deploymentReadiness(spec).Readiness.Type)	285		assert.Equal(t, domain.AppReadinessHTTP, deploymentReadiness(spec).Readiness.Type)
246		})	286		})
247			287
248	-	t.Run("an explicit none on an L4 service stays unchecked", func(t *testing.T) {	288	+	t.Run("an explicit none on a stateful L4 service stays unchecked", func(t *testing.T) {
249		spec := domain.AppService{	289		spec := domain.AppService{
250		Name: "db",	290		Name: "db",
251		TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 5432, Publish: "5432"}},	291		TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 5432, Publish: "5432"}},
			292	+	Volumes: []domain.AppVolume{{Name: "data", Path: "/var/lib/postgresql/data"}},
252		Readiness: domain.AppReadiness{Type: domain.AppReadinessNone},	293		Readiness: domain.AppReadiness{Type: domain.AppReadinessNone},
253		}	294		}
254		assert.Equal(t, domain.AppReadinessNone, deploymentReadiness(spec).Readiness.Type)	295		assert.Equal(t, domain.AppReadinessNone, deploymentReadiness(spec).Readiness.Type)

M internal/usecase/deployment/candidate.go +9 -1


@@ -8,6 +8,7 @@ import (
8	8		"slices"
9	9		"sort"
10	10		"strings"
	11	+	"time"
11	12
12	13		"github.com/bnema/zerowrap"
13	14

@@ -20,6 +21,9 @@ type startedCandidate struct {
20	21		Container *domain.Container
21	22		TCPBinds map[int]int
22	23		UDPBinds map[int]int
	24	+	// StartedAt is the execution start observed right after the container
	25	+	// was started, before readiness ran. Zero when it could not be read.
	26	+	StartedAt time.Time
23	27		}
24	28
25	29		// createAndStart builds the runtime config and starts the replacement.

@@ -123,12 +127,16 @@ func (s *Service) createAndStart(ctx context.Context, app, revision string, p pi
123	127		s.retireCandidate(ctx, app, p.spec.Name, created.ID)
124	128		return startedCandidate{Container: created}, fmt.Errorf("deployment: start container: %w", err)
125	129		}
	130	+	// The execution is observed before readiness runs: a native restart
	131	+	// between readiness and the verified-execution record then shows up as a
	132	+	// different start time, so it is still treated as a fresh execution.
	133	+	startedAt := s.executionStart(ctx, created.ID)
126	134		binds, udpBinds, err := s.readBackendBinds(ctx, app, p.spec.Name, created.ID, backendPorts(p.spec))
127	135		if err != nil {
128	136		s.retireCandidate(ctx, app, p.spec.Name, created.ID)
129	137		return startedCandidate{Container: created}, err
130	138		}
131		-	return startedCandidate{Container: created, TCPBinds: binds, UDPBinds: udpBinds}, nil
	139	+	return startedCandidate{Container: created, TCPBinds: binds, UDPBinds: udpBinds, StartedAt: startedAt}, nil
132	140		}
133	141
134	142		// runtimeImageRef resolves the exact runtime image reference of one pinned

 	"slices"
 	"sort"
 	"strings"
+	"time"
 	"github.com/bnema/zerowrap"
 	Container *domain.Container
 	TCPBinds  map[int]int
 	UDPBinds  map[int]int
+	// StartedAt is the execution start observed right after the container
+	// was started, before readiness ran. Zero when it could not be read.
+	StartedAt time.Time
 }
 // createAndStart builds the runtime config and starts the replacement.
 		s.retireCandidate(ctx, app, p.spec.Name, created.ID)
 		return startedCandidate{Container: created}, fmt.Errorf("deployment: start container: %w", err)
 	}
+	// The execution is observed before readiness runs: a native restart
+	// between readiness and the verified-execution record then shows up as a
+	// different start time, so it is still treated as a fresh execution.
+	startedAt := s.executionStart(ctx, created.ID)
 	binds, udpBinds, err := s.readBackendBinds(ctx, app, p.spec.Name, created.ID, backendPorts(p.spec))
 	if err != nil {
 		s.retireCandidate(ctx, app, p.spec.Name, created.ID)
 		return startedCandidate{Container: created}, err
 	}
-	return startedCandidate{Container: created, TCPBinds: binds, UDPBinds: udpBinds}, nil
+	return startedCandidate{Container: created, TCPBinds: binds, UDPBinds: udpBinds, StartedAt: startedAt}, nil
 }
 // runtimeImageRef resolves the exact runtime image reference of one pinned

M internal/usecase/deployment/crash_recovery_test.go +43 -32


@@ -81,6 +81,7 @@ func TestDeploy_CandidateIsJournaledBeforeItStarts(t *testing.T) {
81	81		journaledBeforeStart = found && step.After == "c-new" && step.State == domain.AppStepPending
82	82		return nil
83	83		}).Once()
	84	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
84	85		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
85	86		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
86	87

@@ -109,8 +110,8 @@ func TestDeploy_CandidateIsJournaledBeforeItStarts(t *testing.T) {
109	110
110	111		// crashDuringReplacement runs one replacement deploy and interrupts it
111	112		// between creating the candidate and publishing it. The candidate is left
112		-	// running, ACTIVE still names the superseded generation, and the operation
113		-	// journal stays in flight: the state a process crash or a failed cleanup
	113	+	// running, ACTIVE still names the superseded generation (which keeps running,
	114	+	// as an overlap replacement leaves it), and the operation journal stays in flight: the state a process crash or a failed cleanup
114	115		// leaves behind.
115	116		func crashDuringReplacement(t testing.T, store appstate.Store, opKey string) {
116	117		t.Helper()

@@ -123,10 +124,10 @@ func crashDuringReplacement(t testing.T, store appstate.Store, opKey string) {
123	124		images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
124	125		runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
125	126		expectNetworkProvision(runtime, "app-blog", 1)
126		-	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
127		-	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	127	+	// The overlap replacement leaves the superseded container untouched.
128	128		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-orphan", Name: "web"}, nil).Once()
129	129		runtime.EXPECT().StartContainer(mock.Anything, "c-orphan").Return(nil).Once()
	130	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-orphan").Return(&domain.Container{ID: "c-orphan", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
130	131		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-orphan", mock.Anything).Return(
131	132		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
132	133		// The canceled context also refuses the candidate cleanup, so the

@@ -165,12 +166,12 @@ func crashDuringReplacement(t testing.T, store appstate.Store, opKey string) {
165	166		require.Error(t, <-done)
166	167		}
167	168
168		-	// TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished
169		-	// proves the crash window between creating a replacement and publishing it is
170		-	// recoverable: an interruption leaves the candidate journaled but unpublished,
171		-	// and boot recovery removes that candidate before it rebuilds the generation
172		-	// recorded in ACTIVE, so two generations of one service never run together.
173		-	func TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished(t *testing.T) {
	169	+	// TestBootRecovery_RemovesUnpublishedCandidateAndKeepsServingGeneration
	170	+	// proves the crash window between creating an overlap replacement and
	171	+	// publishing it is recoverable: an interruption leaves the candidate
	172	+	// journaled but unpublished, and boot recovery removes that candidate and
	173	+	// keeps the generation recorded in ACTIVE, which was never stopped.
	174	+	func TestBootRecovery_RemovesUnpublishedCandidateAndKeepsServingGeneration(t *testing.T) {
174	175		ctx := context.Background()
175	176		store := newTestStore(t)
176	177		images := outmocks.NewMockImageResolver(t)

@@ -192,8 +193,8 @@ func TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished(t *te
192	193		require.True(t, ok)
193	194		assert.Equal(t, "c-old", active.Services["web"].Container)
194	195
195		-	// Boot recovery: the orphan is removed before the published generation is
196		-	// rebuilt from its pinned revision.
	196	+	// Boot recovery: the orphan is removed; the published generation is
	197	+	// still running, so it is verified and kept, never rebuilt or stopped.
197	198		var order []string
198	199		runtime2 := outmocks.NewMockContainerRuntime(t)
199	200		runtime2.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).RunAndReturn(

@@ -201,19 +202,14 @@ func TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished(t *te
201	202		order = append(order, "remove-orphan")
202	203		return nil
203	204		}).Once()
204		-	runtime2.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(false, nil).Once()
205		-	runtime2.EXPECT().StartContainer(mock.Anything, "c-old").Return(domain.ErrContainerNotFound).Once()
206		-	runtime2.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
207		-	runtime2.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
208		-	expectNetworkProvision(runtime2, "app-blog", 1)
209		-	runtime2.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
210		-	func(context.Context, domain.ContainerConfig) (domain.Container, error) {
211		-	order = append(order, "rebuild-published-generation")
212		-	return &domain.Container{ID: "c-rebuilt", Name: "web"}, nil
	205	+	runtime2.EXPECT().IsContainerRunning(mock.Anything, "c-old").RunAndReturn(
	206	+	func(context.Context, string) (bool, error) {
	207	+	order = append(order, "verify-published-generation")
	208	+	return true, nil
213	209		}).Once()
214		-	runtime2.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
215		-	runtime2.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
216		-	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
	210	+	runtime2.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
	211	+	runtime2.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
	212	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
217	213
218	214		rebooted := deployment.NewService(deployment.Deps{
219	215		State: store, Runtime: runtime2, Images: images, Secrets: secrets,

@@ -225,8 +221,11 @@ func TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished(t *te
225	221		// context is gone with it.
226	222		require.NoError(t, rebooted.ReconcileBoot(ctx))
227	223
228		-	assert.Equal(t, []string{"remove-orphan", "rebuild-published-generation"}, order,
229		-	"the unpublished candidate is gone before the published generation is rebuilt")
	224	+	assert.Equal(t, []string{"remove-orphan", "verify-published-generation"}, order,
	225	+	"the unpublished candidate is gone and the serving generation is kept")
	226	+	runtime2.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
	227	+	runtime2.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", mock.Anything)
	228	+	runtime2.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
230	229		runtime2.AssertNotCalled(t, "StartContainer", mock.Anything, "c-orphan")
231	230		runtime2.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-orphan", mock.Anything)
232	231

@@ -241,12 +240,11 @@ func TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished(t *te
241	240		assert.Equal(t, domain.AppStepFailed, finalStep.State)
242	241		assert.Contains(t, finalStep.Error, "unpublished candidate was removed")
243	242
244		-	// ACTIVE now names the rebuilt generation: recovery converged the app
245		-	// instead of leaving it pointing at a container that no longer exists.
	243	+	// ACTIVE still names the generation that kept serving.
246	244		converged, ok, err := store.LoadActive(context.Background(), "blog")
247	245		require.NoError(t, err)
248	246		require.True(t, ok)
249		-	assert.Equal(t, "c-rebuilt", converged.Services["web"].Container)
	247	+	assert.Equal(t, "c-old", converged.Services["web"].Container)
250	248		assert.Equal(t, "rev-0", converged.Services["web"].EffectiveRevision)
251	249		}
252	250

@@ -272,15 +270,22 @@ func TestDeploy_ReconcilesInterruptedPredecessorBeforeCreatingAReplacement(t *te
272	270		}).Once()
273	271		images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
274	272		runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
275		-	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
276		-	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
277	273		expectNetworkProvision(runtime, "app-blog", 1)
278	274		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
279	275		func(context.Context, domain.ContainerConfig) (domain.Container, error) {
280	276		order = append(order, "create-new-generation")
281	277		return &domain.Container{ID: "c-next", Name: "web"}, nil
282	278		}).Once()
	279	+	// The overlap replacement retires the superseded container only after
	280	+	// the new generation is published.
	281	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(
	282	+	func(context.Context, string, time.Duration) error {
	283	+	order = append(order, "retire-superseded")
	284	+	return nil
	285	+	}).Once()
	286	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
283	287		runtime.EXPECT().StartContainer(mock.Anything, "c-next").Return(nil).Once()
	288	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-next").Return(&domain.Container{ID: "c-next", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
284	289		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-next", mock.Anything).Return(
285	290		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
286	291

@@ -295,7 +300,7 @@ func TestDeploy_ReconcilesInterruptedPredecessorBeforeCreatingAReplacement(t *te
295	300		require.NoError(t, err)
296	301		require.NotNil(t, result)
297	302		assert.Equal(t, "deployed", result.Services["web"].Result)
298		-	assert.Equal(t, []string{"remove-leftover-candidate", "create-new-generation"}, order)
	303	+	assert.Equal(t, []string{"remove-leftover-candidate", "create-new-generation", "retire-superseded"}, order)
299	304
300	305		active, ok, err := store.LoadActive(ctx, "blog")
301	306		require.NoError(t, err)

@@ -378,12 +383,14 @@ func TestRestart_FailedRebuildClearsTheInhibitionOfTheMissingContainer(t *testin
378	383		seedRevision(t, ctx, store, "intent-0", "", rev)
379	384
380	385		runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
	386	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
381	387		runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
382	388		runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
383	389		expectNetworkProvision(runtime, "app-blog", 1)
384	390		runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
385	391		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuild", Name: "web"}, nil).Once()
386	392		runtime.EXPECT().StartContainer(mock.Anything, "c-rebuild").Return(nil).Once()
	393	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuild").Return(&domain.Container{ID: "c-rebuild", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
387	394		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuild", mock.Anything).Return(
388	395		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
389	396		runtime.EXPECT().GetContainerLogs(mock.Anything, "c-rebuild", false).Return(nil, assert.AnError).Once()

@@ -580,12 +587,14 @@ func TestBootRecovery_ClearsReplacementInhibitionAndRebuildsFromActive(t *testin
580	587		runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(nil).Once()
581	588		runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(false, nil).Once()
582	589		runtime.EXPECT().StartContainer(mock.Anything, "c-old").Return(domain.ErrContainerNotFound).Once()
	590	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
583	591		runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
584	592		runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
585	593		expectNetworkProvision(runtime, "app-blog", 1)
586	594		runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
587	595		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
588	596		runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
	597	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuilt").Return(&domain.Container{ID: "c-rebuilt", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
589	598		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
590	599		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
591	600

@@ -625,11 +634,13 @@ func TestRestart_RebuildsMissingActiveContainer(t *testing.T) {
625	634		seedReplaceableApp(t, ctx, store)
626	635
627	636		runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
	637	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
628	638		runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
629	639		runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
630	640		expectNetworkProvision(runtime, "app-blog", 1)
631	641		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
632	642		runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
	643	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuilt").Return(&domain.Container{ID: "c-rebuilt", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
633	644		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
634	645		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
635	646

 		journaledBeforeStart = found && step.After == "c-new" && step.State == domain.AppStepPending
 		return nil
 	}).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
 		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 // crashDuringReplacement runs one replacement deploy and interrupts it
 // between creating the candidate and publishing it. The candidate is left
-// running, ACTIVE still names the superseded generation, and the operation
+// running, ACTIVE still names the superseded generation (which keeps running,
-// journal stays in flight: the state a process crash or a failed cleanup
+// as an overlap replacement leaves it), and the operation journal stays in flight: the state a process crash or a failed cleanup
 // leaves behind.
 func crashDuringReplacement(t *testing.T, store *appstate.Store, opKey string) {
 	t.Helper()
 	images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
 	runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
 	expectNetworkProvision(runtime, "app-blog", 1)
-	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
+	// The overlap replacement leaves the superseded container untouched.
-	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-orphan", Name: "web"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-orphan").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-orphan").Return(&domain.Container{ID: "c-orphan", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-orphan", mock.Anything).Return(
 		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	// The canceled context also refuses the candidate cleanup, so the
 	require.Error(t, <-done)
 }
-// TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished
+// TestBootRecovery_RemovesUnpublishedCandidateAndKeepsServingGeneration
-// proves the crash window between creating a replacement and publishing it is
+// proves the crash window between creating an overlap replacement and
-// recoverable: an interruption leaves the candidate journaled but unpublished,
+// publishing it is recoverable: an interruption leaves the candidate
-// and boot recovery removes that candidate before it rebuilds the generation
+// journaled but unpublished, and boot recovery removes that candidate and
-// recorded in ACTIVE, so two generations of one service never run together.
+// keeps the generation recorded in ACTIVE, which was never stopped.
-func TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished(t *testing.T) {
+func TestBootRecovery_RemovesUnpublishedCandidateAndKeepsServingGeneration(t *testing.T) {
 	ctx := context.Background()
 	store := newTestStore(t)
 	images := outmocks.NewMockImageResolver(t)
 	require.True(t, ok)
 	assert.Equal(t, "c-old", active.Services["web"].Container)
-	// Boot recovery: the orphan is removed before the published generation is
+	// Boot recovery: the orphan is removed; the published generation is
-	// rebuilt from its pinned revision.
+	// still running, so it is verified and kept, never rebuilt or stopped.
 	var order []string
 	runtime2 := outmocks.NewMockContainerRuntime(t)
 	runtime2.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).RunAndReturn(
 			order = append(order, "remove-orphan")
 			return nil
 		}).Once()
-	runtime2.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(false, nil).Once()
+	runtime2.EXPECT().IsContainerRunning(mock.Anything, "c-old").RunAndReturn(
-	runtime2.EXPECT().StartContainer(mock.Anything, "c-old").Return(domain.ErrContainerNotFound).Once()
+		func(context.Context, string) (bool, error) {
-	runtime2.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
+			order = append(order, "verify-published-generation")
-	runtime2.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
+			return true, nil
-	expectNetworkProvision(runtime2, "app-blog", 1)
-	runtime2.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
-		func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
-			order = append(order, "rebuild-published-generation")
-			return &domain.Container{ID: "c-rebuilt", Name: "web"}, nil
 		}).Once()
-	runtime2.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
+	runtime2.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
-	runtime2.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
+	runtime2.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
-		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
+		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	rebooted := deployment.NewService(deployment.Deps{
 		State: store, Runtime: runtime2, Images: images, Secrets: secrets,
 	// context is gone with it.
 	require.NoError(t, rebooted.ReconcileBoot(ctx))
-	assert.Equal(t, []string{"remove-orphan", "rebuild-published-generation"}, order,
+	assert.Equal(t, []string{"remove-orphan", "verify-published-generation"}, order,
-		"the unpublished candidate is gone before the published generation is rebuilt")
+		"the unpublished candidate is gone and the serving generation is kept")
+	runtime2.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
+	runtime2.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", mock.Anything)
+	runtime2.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
 	runtime2.AssertNotCalled(t, "StartContainer", mock.Anything, "c-orphan")
 	runtime2.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-orphan", mock.Anything)
 	assert.Equal(t, domain.AppStepFailed, finalStep.State)
 	assert.Contains(t, finalStep.Error, "unpublished candidate was removed")
-	// ACTIVE now names the rebuilt generation: recovery converged the app
+	// ACTIVE still names the generation that kept serving.
-	// instead of leaving it pointing at a container that no longer exists.
 	converged, ok, err := store.LoadActive(context.Background(), "blog")
 	require.NoError(t, err)
 	require.True(t, ok)
-	assert.Equal(t, "c-rebuilt", converged.Services["web"].Container)
+	assert.Equal(t, "c-old", converged.Services["web"].Container)
 	assert.Equal(t, "rev-0", converged.Services["web"].EffectiveRevision)
 }
 		}).Once()
 	images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
 	runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
-	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
-	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
 	expectNetworkProvision(runtime, "app-blog", 1)
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
 		func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
 			order = append(order, "create-new-generation")
 			return &domain.Container{ID: "c-next", Name: "web"}, nil
 		}).Once()
+	// The overlap replacement retires the superseded container only after
+	// the new generation is published.
+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(
+		func(context.Context, string, time.Duration) error {
+			order = append(order, "retire-superseded")
+			return nil
+		}).Once()
+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-next").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-next").Return(&domain.Container{ID: "c-next", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-next", mock.Anything).Return(
 		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	require.NoError(t, err)
 	require.NotNil(t, result)
 	assert.Equal(t, "deployed", result.Services["web"].Result)
-	assert.Equal(t, []string{"remove-leftover-candidate", "create-new-generation"}, order)
+	assert.Equal(t, []string{"remove-leftover-candidate", "create-new-generation", "retire-superseded"}, order)
 	active, ok, err := store.LoadActive(ctx, "blog")
 	require.NoError(t, err)
 	seedRevision(t, ctx, store, "intent-0", "", rev)
 	runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
 	expectNetworkProvision(runtime, "app-blog", 1)
 	runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuild", Name: "web"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-rebuild").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuild").Return(&domain.Container{ID: "c-rebuild", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuild", mock.Anything).Return(
 		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	runtime.EXPECT().GetContainerLogs(mock.Anything, "c-rebuild", false).Return(nil, assert.AnError).Once()
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(nil).Once()
 	runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(false, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-old").Return(domain.ErrContainerNotFound).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
 	expectNetworkProvision(runtime, "app-blog", 1)
 	runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuilt").Return(&domain.Container{ID: "c-rebuilt", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
 		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	seedReplaceableApp(t, ctx, store)
 	runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
 	expectNetworkProvision(runtime, "app-blog", 1)
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuilt").Return(&domain.Container{ID: "c-rebuilt", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
 		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()

M internal/usecase/deployment/deploy_mock_test.go +104 -29


@@ -166,6 +166,7 @@ func TestDeploy_Mockery_PullsPinnedImageWithRegistryAuth(t *testing.T) {
166	166		assert.Equal(t, svcSpec.Command, cfg.Entrypoint) && assert.Empty(t, cfg.Cmd)
167	167		})).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
168	168		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	169	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
169	170		// No TCP-capable interfaces (HTTP nil, readiness empty): no backend
170	171		// binds published, no GetContainerBackendBinds call.
171	172		state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{

@@ -216,6 +217,7 @@ func TestDeploy_Mockery_HTTPSuccessRecordsLoopbackBinds(t *testing.T) {
216	217
217	218		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
218	219		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	220	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
219	221		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
220	222		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
221	223		return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"

@@ -246,25 +248,101 @@ func TestDeploy_Mockery_HTTPSuccessRecordsLoopbackBinds(t *testing.T) {
246	248		runtime.AssertNotCalled(t, "GetContainerNetworkInfo", mock.Anything, mock.Anything)
247	249		}
248	250
249		-	// TestDeploy_Mockery_HTTPReplacesOldGenerationBeforePublish proves the
250		-	// sequential ordering: the superseded container is stopped and removed
251		-	// before the replacement is created, and ACTIVE names the replacement only
252		-	// after it exists. A public HTTP service without volumes is replaced exactly
253		-	// like every other service.
254		-	func TestDeploy_Mockery_HTTPReplacesOldGenerationBeforePublish(t *testing.T) {
	251	+	// TestDeploy_Mockery_HTTPOverlapPublishesBeforeRetiringOldGeneration proves
	252	+	// the overlap ordering of a stateless public HTTP service: the replacement is
	253	+	// created while the superseded container keeps running, ACTIVE names the
	254	+	// replacement next, and the superseded container is stopped and removed only
	255	+	// afterwards, releasing its backend claims.
	256	+	func TestDeploy_Mockery_HTTPOverlapPublishesBeforeRetiringOldGeneration(t *testing.T) {
255	257		ctx := context.Background()
256	258		state, runtime, images, secrets := mockDeps(t)
257	259		rev := mockRevision()
258	260
259	261		expectDeployPreflight(state, runtime, images, secrets, rev)
260	262		expectNetworkProvision(runtime, "app-blog", 1)
261		-	state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
	263	+	oldActive := domain.AppActive{
	264	+	App: "blog",
	265	+	Services: map[string]domain.AppEffectiveService{
	266	+	"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", BackendBinds: map[int]int{8080: 17000}},
	267	+	},
	268	+	}
	269	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(oldActive, true, nil)
	270	+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	271	+
	272	+	var createdAt, publishedAt, stoppedAt int
	273	+	var step int
	274	+	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(func(context.Context, domain.ContainerConfig) (domain.Container, error) {
	275	+	step++
	276	+	createdAt = step
	277	+	return &domain.Container{ID: "c-new", Name: "web"}, nil
	278	+	}).Once()
	279	+	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	280	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
	281	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
	282	+	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
	283	+	return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
	284	+	})).Return(nil).Once()
	285	+	var published domain.AppActive
	286	+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, active domain.AppActive) error {
	287	+	step++
	288	+	publishedAt = step
	289	+	published = active
	290	+	return nil
	291	+	}).Once()
	292	+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
	293	+	expectRetirementMarker(state, "c-old")
	294	+	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
	295	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(func(context.Context, string, time.Duration) error {
	296	+	step++
	297	+	stoppedAt = step
	298	+	return nil
	299	+	}).Once()
	300	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	301	+	// The superseded container's loopback claims are released once it is gone.
	302	+	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
	303	+
	304	+	svc := deployment.NewService(
	305	+	deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
	306	+	zerowrap.Default(),
	307	+	).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	308	+	func(context.Context, string, string) (int, error) { return 200, nil },
	309	+	func(context.Context, string) error { return nil },
	310	+	))
	311	+
	312	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	313	+	require.NoError(t, err)
	314	+	require.NotNil(t, result)
	315	+	assert.Equal(t, "deployed", result.Services["web"].Result)
	316	+	assert.Less(t, createdAt, publishedAt, "ACTIVE is published after the replacement exists")
	317	+	assert.Less(t, publishedAt, stoppedAt, "the superseded generation is stopped only after the switch")
	318	+	assert.Equal(t, "c-new", published.Services["web"].Container)
	319	+	assert.Equal(t, map[int]int{8080: 18080}, published.Services["web"].BackendBinds,
	320	+	"ACTIVE no longer references the superseded container's binds")
	321	+	assert.Empty(t, result.CleanupWarnings)
	322	+	}
	323	+
	324	+	// TestDeploy_Mockery_HTTPSequentialWhenServiceHasVolume proves a service with a
	325	+	// volume keeps the sequential ordering: the superseded container is stopped
	326	+	// and removed before the replacement is created, and ACTIVE names the
	327	+	// replacement only after it exists.
	328	+	func TestDeploy_Mockery_HTTPSequentialWhenServiceHasVolume(t *testing.T) {
	329	+	ctx := context.Background()
	330	+	state, runtime, images, secrets := mockDeps(t)
	331	+	rev := statefulMockRevision()
	332	+
	333	+	expectDeployPreflight(state, runtime, images, secrets, rev)
	334	+	runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil)
	335	+	runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil)
	336	+	expectNetworkProvision(runtime, "app-blog", 1)
	337	+	active := domain.AppActive{
262	338		App: "blog",
263	339		Services: map[string]domain.AppEffectiveService{
264	340		"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old"},
265	341		},
266		-	}, true, nil)
	342	+	}
	343	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
267	344		state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	345	+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
268	346
269	347		var replacedAt, createdAt, publishedAt int
270	348		var step int

@@ -281,16 +359,9 @@ func TestDeploy_Mockery_HTTPReplacesOldGenerationBeforePublish(t *testing.T) {
281	359		return &domain.Container{ID: "c-new", Name: "web"}, nil
282	360		}).Once()
283	361		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	362	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
284	363		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
285		-	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
286		-	return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
287		-	})).Return(nil).Once()
288		-	state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
289		-	App: "blog",
290		-	Services: map[string]domain.AppEffectiveService{
291		-	"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old"},
292		-	},
293		-	}, true, nil)
	364	+	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
294	365		state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(func(context.Context, domain.AppActive) error {
295	366		step++
296	367		publishedAt = step

@@ -318,11 +389,11 @@ func TestDeploy_Mockery_HTTPReplacesOldGenerationBeforePublish(t *testing.T) {
318	389		assert.Empty(t, result.CleanupWarnings)
319	390		}
320	391
321		-	// TestDeploy_Mockery_HTTPReadinessFailureLeavesNoOldGeneration proves a
322		-	// failed replacement never falls back to the generation it already retired:
323		-	// the superseded container stays gone, the unready candidate is removed, and
324		-	// nothing is published in its place.
325		-	func TestDeploy_Mockery_HTTPReadinessFailureLeavesNoOldGeneration(t *testing.T) {
	392	+	// TestDeploy_Mockery_HTTPReadinessFailureKeepsOldGenerationServing proves a
	393	+	// failed overlap replacement leaves the generation that serves alone: the
	394	+	// superseded container is neither stopped nor removed nor released, the
	395	+	// unready candidate is removed, and nothing is published in its place.
	396	+	func TestDeploy_Mockery_HTTPReadinessFailureKeepsOldGenerationServing(t *testing.T) {
326	397		ctx := context.Background()
327	398		state, runtime, images, secrets := mockDeps(t)
328	399		rev := mockRevision()

@@ -337,13 +408,9 @@ func TestDeploy_Mockery_HTTPReadinessFailureLeavesNoOldGeneration(t *testing.T)
337	408		}, true, nil)
338	409		state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
339	410
340		-	// The superseded generation is retired before the replacement is created,
341		-	// then the replacement fails readiness fast.
342		-	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
343		-	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
344		-	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
345	411		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
346	412		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	413	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
347	414		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
348	415		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
349	416		return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"

@@ -369,10 +436,12 @@ func TestDeploy_Mockery_HTTPReadinessFailureLeavesNoOldGeneration(t *testing.T)
369	436		assert.NotContains(t, result.Services["web"].Error, "logs:", "raw logs must never be embedded in the public error")
370	437		require.Len(t, result.Services["web"].Diagnostics, 1, "failure diagnostics are kept separately")
371	438		assert.Equal(t, "boom", result.Services["web"].Diagnostics[0])
372		-	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-new", mock.Anything)
	439	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
	440	+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", mock.Anything)
373	441		runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
374	442		state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
375		-	state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
	443	+	state.AssertNotCalled(t, "SaveRecoveryInhibition", mock.Anything, mock.Anything)
	444	+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-old")
376	445		}
377	446
378	447		func TestDeploy_Mockery_VolumeReplacementMarksUnsafe(t *testing.T) {

@@ -459,6 +528,7 @@ func TestDeploy_Mockery_VolumeReplacementMarksUnsafe(t *testing.T) {
459	528		return &domain.Container{ID: "c-new", Name: "web"}, nil
460	529		}).Once()
461	530		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	531	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
462	532		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 9000, HostPort: 19000, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
463	533		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
464	534		return len(claims) == 1 && claims[0].Port == 19000 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"

@@ -537,6 +607,7 @@ func TestDeploy_Mockery_MixedTCPUDPPublishesBothLoopbacks(t *testing.T) {
537	607		return len(byProto) == 2
538	608		})).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
539	609		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	610	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
540	611		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.MatchedBy(func(ports []domain.ContainerBackendPort) bool {
541	612		if len(ports) != 2 {
542	613		return false

@@ -641,6 +712,7 @@ func TestDeploy_Mockery_InjectsAppEnvAndSecrets(t *testing.T) {
641	712		return len(cfg.Env) == 2
642	713		})).Return(&domain.Container{ID: "c-worker", Name: "worker"}, nil).Once()
643	714		runtime.EXPECT().StartContainer(mock.Anything, mock.Anything).Return(nil)
	715	+	runtime.EXPECT().InspectContainer(mock.Anything, mock.Anything).Return(&domain.Container{Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
644	716		state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
645	717		App: "blog", Services: map[string]domain.AppEffectiveService{},
646	718		}, true, nil)

@@ -698,6 +770,7 @@ func TestStart_RunningContainerRefreshesShiftedBinds(t *testing.T) {
698	770		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
699	771		runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(true, nil)
700	772		// Native restart shifted the ephemeral bind: 32770 -> 32771.
	773	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
701	774		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
702	775		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
703	776		return len(claims) == 1 && claims[0].Port == 32771 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-old"

@@ -753,6 +826,7 @@ func TestStart_BindVerificationFailureFailsClosed(t *testing.T) {
753	826		state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
754	827		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
755	828		runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(true, nil)
	829	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
756	830		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(nil, assert.AnError).Once()
757	831
758	832		traffic := &recordingTraffic{}

@@ -803,6 +877,7 @@ func TestRestart_MixedServiceRefreshesBothProtocols(t *testing.T) {
803	877		state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
804	878		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
805	879		runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	880	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
806	881		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return([]domain.ContainerBackendBind{
807	882		{ContainerPort: 9000, HostPort: 32771, Protocol: domain.NetworkProtocolTCP},
808	883		{ContainerPort: 9000, HostPort: 32781, Protocol: domain.NetworkProtocolUDP},

 			assert.Equal(t, svcSpec.Command, cfg.Entrypoint) && assert.Empty(t, cfg.Cmd)
 	})).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	// No TCP-capable interfaces (HTTP nil, readiness empty): no backend
 	// binds published, no GetContainerBackendBinds call.
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
 		return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
 	runtime.AssertNotCalled(t, "GetContainerNetworkInfo", mock.Anything, mock.Anything)
 }
-// TestDeploy_Mockery_HTTPReplacesOldGenerationBeforePublish proves the
+// TestDeploy_Mockery_HTTPOverlapPublishesBeforeRetiringOldGeneration proves
-// sequential ordering: the superseded container is stopped and removed
+// the overlap ordering of a stateless public HTTP service: the replacement is
-// before the replacement is created, and ACTIVE names the replacement only
+// created while the superseded container keeps running, ACTIVE names the
-// after it exists. A public HTTP service without volumes is replaced exactly
+// replacement next, and the superseded container is stopped and removed only
-// like every other service.
+// afterwards, releasing its backend claims.
-func TestDeploy_Mockery_HTTPReplacesOldGenerationBeforePublish(t *testing.T) {
+func TestDeploy_Mockery_HTTPOverlapPublishesBeforeRetiringOldGeneration(t *testing.T) {
 	ctx := context.Background()
 	state, runtime, images, secrets := mockDeps(t)
 	rev := mockRevision()
 	expectDeployPreflight(state, runtime, images, secrets, rev)
 	expectNetworkProvision(runtime, "app-blog", 1)
-	state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
+	oldActive := domain.AppActive{
+		App: "blog",
+		Services: map[string]domain.AppEffectiveService{
+			"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", BackendBinds: map[int]int{8080: 17000}},
+		},
+	}
+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(oldActive, true, nil)
+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
+	var createdAt, publishedAt, stoppedAt int
+	var step int
+	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
+		step++
+		createdAt = step
+		return &domain.Container{ID: "c-new", Name: "web"}, nil
+	}).Once()
+	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
+	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
+		return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
+	})).Return(nil).Once()
+	var published domain.AppActive
+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, active domain.AppActive) error {
+		step++
+		publishedAt = step
+		published = active
+		return nil
+	}).Once()
+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
+	expectRetirementMarker(state, "c-old")
+	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(func(context.Context, string, time.Duration) error {
+		step++
+		stoppedAt = step
+		return nil
+	}).Once()
+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
+	// The superseded container's loopback claims are released once it is gone.
+	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
+	svc := deployment.NewService(
+		deployment.Deps{State: state, Runtime: runtime, Images: images, Secrets: secrets},
+		zerowrap.Default(),
+	).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
+		func(context.Context, string, string) (int, error) { return 200, nil },
+		func(context.Context, string) error { return nil },
+	))
+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	assert.Equal(t, "deployed", result.Services["web"].Result)
+	assert.Less(t, createdAt, publishedAt, "ACTIVE is published after the replacement exists")
+	assert.Less(t, publishedAt, stoppedAt, "the superseded generation is stopped only after the switch")
+	assert.Equal(t, "c-new", published.Services["web"].Container)
+	assert.Equal(t, map[int]int{8080: 18080}, published.Services["web"].BackendBinds,
+		"ACTIVE no longer references the superseded container's binds")
+	assert.Empty(t, result.CleanupWarnings)
+}
+// TestDeploy_Mockery_HTTPSequentialWhenServiceHasVolume proves a service with a
+// volume keeps the sequential ordering: the superseded container is stopped
+// and removed before the replacement is created, and ACTIVE names the
+// replacement only after it exists.
+func TestDeploy_Mockery_HTTPSequentialWhenServiceHasVolume(t *testing.T) {
+	ctx := context.Background()
+	state, runtime, images, secrets := mockDeps(t)
+	rev := statefulMockRevision()
+	expectDeployPreflight(state, runtime, images, secrets, rev)
+	runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil)
+	runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	expectNetworkProvision(runtime, "app-blog", 1)
+	active := domain.AppActive{
 		App: "blog",
 		Services: map[string]domain.AppEffectiveService{
 			"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old"},
 		},
-	}, true, nil)
+	}
+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
 	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
 	var replacedAt, createdAt, publishedAt int
 	var step int
 		return &domain.Container{ID: "c-new", Name: "web"}, nil
 	}).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
-	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
+	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
-		return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
-	})).Return(nil).Once()
-	state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
-		App: "blog",
-		Services: map[string]domain.AppEffectiveService{
-			"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old"},
-		},
-	}, true, nil)
 	state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(func(context.Context, domain.AppActive) error {
 		step++
 		publishedAt = step
 	assert.Empty(t, result.CleanupWarnings)
 }
-// TestDeploy_Mockery_HTTPReadinessFailureLeavesNoOldGeneration proves a
+// TestDeploy_Mockery_HTTPReadinessFailureKeepsOldGenerationServing proves a
-// failed replacement never falls back to the generation it already retired:
+// failed overlap replacement leaves the generation that serves alone: the
-// the superseded container stays gone, the unready candidate is removed, and
+// superseded container is neither stopped nor removed nor released, the
-// nothing is published in its place.
+// unready candidate is removed, and nothing is published in its place.
-func TestDeploy_Mockery_HTTPReadinessFailureLeavesNoOldGeneration(t *testing.T) {
+func TestDeploy_Mockery_HTTPReadinessFailureKeepsOldGenerationServing(t *testing.T) {
 	ctx := context.Background()
 	state, runtime, images, secrets := mockDeps(t)
 	rev := mockRevision()
 	}, true, nil)
 	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
-	// The superseded generation is retired before the replacement is created,
-	// then the replacement fails readiness fast.
-	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
-	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
-	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
 		return len(claims) == 1 && claims[0].Port == 18080 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
 	assert.NotContains(t, result.Services["web"].Error, "logs:", "raw logs must never be embedded in the public error")
 	require.Len(t, result.Services["web"].Diagnostics, 1, "failure diagnostics are kept separately")
 	assert.Equal(t, "boom", result.Services["web"].Diagnostics[0])
-	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-new", mock.Anything)
+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", mock.Anything)
 	runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
 	state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
-	state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
+	state.AssertNotCalled(t, "SaveRecoveryInhibition", mock.Anything, mock.Anything)
+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-old")
 }
 func TestDeploy_Mockery_VolumeReplacementMarksUnsafe(t *testing.T) {
 		return &domain.Container{ID: "c-new", Name: "web"}, nil
 	}).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 9000, HostPort: 19000, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
 		return len(claims) == 1 && claims[0].Port == 19000 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-new"
 		return len(byProto) == 2
 	})).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.MatchedBy(func(ports []domain.ContainerBackendPort) bool {
 		if len(ports) != 2 {
 			return false
 		return len(cfg.Env) == 2
 	})).Return(&domain.Container{ID: "c-worker", Name: "worker"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, mock.Anything).Return(nil)
+	runtime.EXPECT().InspectContainer(mock.Anything, mock.Anything).Return(&domain.Container{Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
 		App: "blog", Services: map[string]domain.AppEffectiveService{},
 	}, true, nil)
 	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
 	runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(true, nil)
 	// Native restart shifted the ephemeral bind: 32770 -> 32771.
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
 		return len(claims) == 1 && claims[0].Port == 32771 && claims[0].Owner == domain.OwnerGordonBackend && claims[0].ContainerID == "c-old"
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
 	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
 	runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(true, nil)
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(nil, assert.AnError).Once()
 	traffic := &recordingTraffic{}
 	state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
 	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
 	runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return([]domain.ContainerBackendBind{
 		{ContainerPort: 9000, HostPort: 32771, Protocol: domain.NetworkProtocolTCP},
 		{ContainerPort: 9000, HostPort: 32781, Protocol: domain.NetworkProtocolUDP},

M internal/usecase/deployment/deploy_unchanged_test.go +3 -0


@@ -65,6 +65,7 @@ func expectReplacement(runtime *outmocks.MockContainerRuntime) {
65	65		runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
66	66		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
67	67		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	68	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
68	69		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
69	70		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
70	71		}

@@ -179,6 +180,8 @@ func TestDeploy_ReplacesWhenTheDigestChanged(t *testing.T) {
179	180		runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
180	181		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
181	182		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	183	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
	184	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
182	185		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
183	186		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
184	187


@@ -65,6 +65,7 @@ func expectReplacement(runtime *outmocks.MockContainerRuntime) {
65	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()	65		runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
66	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()	66		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
67	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()	67		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
		68	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
68	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(	69		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
69	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()	70		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
70	}	71		}

@@ -179,6 +180,8 @@ func TestDeploy_ReplacesWhenTheDigestChanged(t *testing.T) {
179	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()	180		runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
180	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()	181		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
181	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()	182		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
		183	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
		184	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
182	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(	185		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
183	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()	186		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
184		187

M internal/usecase/deployment/device_policy_internal_test.go +1 -0


@@ -127,6 +127,7 @@ func TestCreateAndStart_PassesResolvedCDIIDsToRuntime(t *testing.T) {
127	127		return assert.Equal(t, []string{"example.com/gpu=GPU-test-uuid"}, cfg.CDIDevices)
128	128		})).Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
129	129		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
	130	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
130	131
131	132		svc := NewService(Deps{State: state, Runtime: runtime, Networks: NetworkConfig{Prefix: "gordon"}}, zerowrap.Default()).
132	133		WithDevicePolicies(map[string]domain.AppDevicePolicy{


@@ -127,6 +127,7 @@ func TestCreateAndStart_PassesResolvedCDIIDsToRuntime(t *testing.T) {
127	return assert.Equal(t, []string{"example.com/gpu=GPU-test-uuid"}, cfg.CDIDevices)	127		return assert.Equal(t, []string{"example.com/gpu=GPU-test-uuid"}, cfg.CDIDevices)
128	})).Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()	128		})).Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
129	runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()	129		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
		130	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
130		131
131	svc := NewService(Deps{State: state, Runtime: runtime, Networks: NetworkConfig{Prefix: "gordon"}}, zerowrap.Default()).	132		svc := NewService(Deps{State: state, Runtime: runtime, Networks: NetworkConfig{Prefix: "gordon"}}, zerowrap.Default()).
132	WithDevicePolicies(map[string]domain.AppDevicePolicy{	133		WithDevicePolicies(map[string]domain.AppDevicePolicy{

M internal/usecase/deployment/execute.go +220 -41


@@ -20,12 +20,16 @@ const failLogTailLines = 50
20	20		const failLogTailTimeout = 10 * time.Second
21	21
22	22		// Deploy executes a preflighted revision: fail-fast across services in
23		-	// sorted name order. Every service is replaced sequentially — withdraw
	23	+	// sorted name order. A service without volumes, binds, devices, or UDP interfaces is
	24	+	// replaced with a short overlap: its replacement starts and becomes ready
	25	+	// while the superseded container keeps serving, traffic switches to it, and
	26	+	// only then is the superseded container stopped, so the service is not
	27	+	// interrupted. Every other service is replaced sequentially — withdraw
24	28		// traffic, retire the superseded container, start the replacement and wait
25		-	// for its readiness probe — so a deployment may briefly interrupt the
26		-	// service, and two Gordon-managed generations of one service never run at
27		-	// the same time. Volumes are never deleted: no RemoveVolume call, no
28		-	// volume-deletion flags on container removal.
	29	+	// for its readiness probe — so a deployment may briefly interrupt it and two
	30	+	// Gordon-managed generations of a single-writer service never run at the same
	31	+	// time. Volumes are never deleted: no RemoveVolume call, no volume-deletion
	32	+	// flags on container removal.
29	33		//
30	34		// It is the synchronous composition of the two engine phases: StartDeploy
31	35		// claims the key and journals the operation, then ExecuteDeploy runs the

@@ -147,7 +151,7 @@ func (s Service) startDeployLocked(ctx context.Context, input DeployInput) (St
147	151
148	152		// ExecuteDeploy performs the second phase of a deploy: it reacquires the app
149	153		// lock, loads and verifies the claimed journal by app/op identity, runs the
150		-	// image pull/pin and the sequential replacement, and records the terminal
	154	+	// image pull/pin and the per-service replacement, and records the terminal
151	155		// outcome. A claim that is already terminal, or that does not match its
152	156		// identity, is never executed: it replays its stored outcome instead. An
153	157		// operation that stops mid-execution (shutdown or cancellation) leaves its

@@ -305,6 +309,24 @@ func (s *Service) claimRevision(ctx context.Context, claim DeployClaim, op domai
305	309		// step is recorded as succeeded only after the runtime effect, the ACTIVE
306	310		// publication, and the traffic apply all succeeded, so a journaled success
307	311		// is never ahead of what is routed.
	312	+	//
	313	+	// An overlap-eligible service (see overlapEligible) is replaced without
	314	+	// withdrawing the superseded container: it keeps serving and is retired only
	315	+	// after the new generation is published and routed. Before the switch the
	316	+	// superseded container gets a durable retirement-pending marker carrying its
	317	+	// stop grace. That marker, not the operation journal, owns the cleanup: it
	318	+	// survives newer operations, and the periodic reconciliation retires the
	319	+	// container once its successor is verified and routed.
	320	+	//
	321	+	// If the publication or the traffic apply of an overlap step fails, the
	322	+	// superseded container is deliberately left running: the proxy may still
	323	+	// route to it, and ACTIVE may already name the new container, so stopping it
	324	+	// could create an outage the failed deploy cannot repair. The step is
	325	+	// journaled as failed with the new container as After and a cleanup warning
	326	+	// names the superseded one. If ACTIVE names the new container, the periodic
	327	+	// reconciliation verifies and republishes it and then retires the superseded
	328	+	// one through its marker; if ACTIVE never changed, it still names the
	329	+	// superseded container, which keeps serving, and the marker is dropped.
308	330		func (s *Service) runServiceStep(
309	331		ctx context.Context,
310	332		app, revision string,

@@ -331,6 +353,10 @@ func (s *Service) runServiceStep(
331	353		Before: before, State: domain.AppStepPending,
332	354		}
333	355		var svcResult ServiceResult
	356	+	overlap := false
	357	+	// The grace of the generation being superseded, read before anything
	358	+	// publishes a record of its replacement.
	359	+	beforeGrace := activeStopGrace(active, p.name)
334	360		if current, ok := s.unchangedService(ctx, app, p, active); ok {
335	361		// Same image, spec, environment, and secret values already running:
336	362		// keep the container.

@@ -341,8 +367,11 @@ func (s *Service) runServiceStep(
341	367		RestartUnsafe: singleWriterRequired(p.spec),
342	368		}
343	369		step.Detail = domain.AppServiceUnchanged + ": already running " + p.spec.Image + " (" + p.digest + ")"
	370	+	} else if before != "" && canOverlap(p.spec, active.Services[p.name].Spec) {
	371	+	overlap = true
	372	+	svcResult = s.deployOverlap(ctx, app, revision, p, op.Op, before, beforeGrace, s.journalCandidate(op, index+1))
344	373		} else {
345		-	svcResult = s.deployService(ctx, app, revision, p, op.Op, before, activeStopGrace(active, p.name), s.journalCandidate(op, index+1))
	374	+	svcResult = s.deployService(ctx, app, revision, p, op.Op, before, beforeGrace, s.journalCandidate(op, index+1))
346	375		}
347	376		result.Services[p.name] = svcResult
348	377		// A recorded candidate is authoritative while the step is unresolved: a

@@ -367,22 +396,22 @@ func (s *Service) runServiceStep(
367	396		return fail(fmt.Errorf("deployment: service %q failed at %s: %s: %w",
368	397		p.name, stepID, svcResult.Error, domain.ErrAppStateConflict))
369	398		}
370		-	// Publish the per-service effective state: the new binds become the
371		-	// service's recorded backend before the proxy is repointed at them.
372		-	if err := s.publishService(ctx, app, revision, p, svcResult, op.Op); err != nil {
373		-	svcResult.Result = "failed"
374		-	svcResult.Error = err.Error()
	399	+	svcResult, err := s.publishAndRoute(ctx, app, revision, p, svcResult, op.Op)
	400	+	if err != nil {
	401	+	if overlap {
	402	+	svcResult.CleanupWarnings = append(svcResult.CleanupWarnings, CleanupWarning{
	403	+	Service: p.name, Leftover: before,
	404	+	Detail: "superseded container kept running: the new generation was not confirmed published and routed",
	405	+	})
	406	+	}
375	407		result.Services[p.name] = svcResult
376	408		return fail(err)
377	409		}
378		-	// Rebuild and publish the traffic graph. ACTIVE already names the new
379		-	// container, but the operation is a failure until routing accepted it,
380		-	// so a rejected graph must never be journaled as success.
381		-	if err := s.refreshTraffic(ctx, app); err != nil {
382		-	svcResult.Result = "failed"
383		-	svcResult.Error = err.Error()
384		-	result.Services[p.name] = svcResult
385		-	return fail(err)
	410	+	// The generation is verified and routed: the periodic pass must not
	411	+	// mistake its first observation for an unseen execution and withdraw it.
	412	+	s.recordVerifiedExecution(app, p.name, svcResult)
	413	+	if overlap {
	414	+	svcResult = s.retireSuperseded(ctx, app, p.name, before, beforeGrace, svcResult)
386	415		}
387	416		result.Services[p.name] = svcResult
388	417		step.State = domain.AppStepSucceeded

@@ -393,6 +422,45 @@ func (s *Service) runServiceStep(
393	422		return nil
394	423		}
395	424
	425	+	// publishAndRoute publishes the per-service effective state, so the new
	426	+	// binds become the service's recorded backend, then rebuilds the traffic
	427	+	// graph so the proxy is repointed at them. ACTIVE already names the new
	428	+	// container when the graph is applied, but the operation is a failure until
	429	+	// routing accepted it, so a rejected graph is returned as a failed result and
	430	+	// never journaled as success.
	431	+	func (s *Service) publishAndRoute(ctx context.Context, app, revision string, p pinnedService, svcResult ServiceResult, opID string) (ServiceResult, error) {
	432	+	if err := s.publishService(ctx, app, revision, p, svcResult, opID); err != nil {
	433	+	svcResult.Result = "failed"
	434	+	svcResult.Error = err.Error()
	435	+	return svcResult, err
	436	+	}
	437	+	if err := s.refreshTraffic(ctx, app); err != nil {
	438	+	svcResult.Result = "failed"
	439	+	svcResult.Error = err.Error()
	440	+	return svcResult, err
	441	+	}
	442	+	return svcResult, nil
	443	+	}
	444	+
	445	+	// retireSuperseded stops the container an overlap replacement superseded,
	446	+	// once the new generation is published and routed. It uses the stop grace of
	447	+	// the generation being stopped: the runtime sends SIGTERM, then kills after
	448	+	// the grace. Backend claims are released and the retirement-pending marker is
	449	+	// cleared by retireContainer once the container is confirmed gone. A
	450	+	// retirement that fails is not a deploy failure — the new generation already
	451	+	// serves — but a cleanup warning on the service; the marker stays, so the
	452	+	// periodic reconciliation retries it.
	453	+	func (s *Service) retireSuperseded(ctx context.Context, app, service, before string, grace time.Duration, result ServiceResult) ServiceResult {
	454	+	retired := s.retireContainer(ctx, app, retireOptions{Service: service, Grace: grace, ClearInhibition: true, RetryOnLeak: true}, before)
	455	+	result.CleanupWarnings = append(result.CleanupWarnings, retired.Warnings...)
	456	+	if !retired.Gone && len(retired.Warnings) == 0 {
	457	+	result.CleanupWarnings = append(result.CleanupWarnings, CleanupWarning{
	458	+	Service: service, Leftover: before, Detail: "superseded container not confirmed gone",
	459	+	})
	460	+	}
	461	+	return result
	462	+	}
	463	+
396	464		// unchangedService reports whether the service already runs exactly what p
397	465		// would create, in a healthy published state: the same digest, service spec,
398	466		// app environment, and shared networks, in a running container that is not

@@ -579,7 +647,10 @@ func (s Service) journalCandidate(op domain.AppOperation, index int) candidate
579	647
580	648		// deployService replaces one service sequentially and returns its terminal
581	649		// result. before is the ACTIVE container being superseded (empty on a first
582		-	// deployment); beforeGrace is its effective stop grace.
	650	+	// deployment); beforeGrace is its effective stop grace. It serves every
	651	+	// service that is not overlap-eligible, a first deployment, and the
	652	+	// rebuild of a missing generation by start and restart; deployOverlap is the
	653	+	// overlapping alternative used by a deploy.
583	654		//
584	655		// The order is fixed: withdraw from traffic and confirm it, durably inhibit
585	656		// a single-writer generation, retire the superseded container and confirm

@@ -591,11 +662,8 @@ func (s *Service) deployService(ctx context.Context, app, revision string, p pin
591	662		// Revalidate the current authorization and source before withdrawing or
592	663		// retiring the serving generation. A reload between preflight and this
593	664		// service step must fail without mutating the existing workload.
594		-	if _, err := s.resolveServiceBinds(app, p.spec); err != nil {
595		-	return s.failResult(revision, before, "", err)
596		-	}
597		-	if _, err := s.resolveServiceDevices(app, p.spec); err != nil {
598		-	return s.failResult(revision, before, "", err)
	665	+	if failed, ok := s.revalidateService(app, revision, p, before); !ok {
	666	+	return failed
599	667		}
600	668		// Withdraw the service from traffic first and confirm it: a withdrawal
601	669		// that cannot be applied must block every container mutation, so no

@@ -629,6 +697,70 @@ func (s *Service) deployService(ctx context.Context, app, revision string, p pin
629	697		return s.failResult(revision, before, "", fmt.Errorf("deployment: retire superseded container %s: %s", before, cleanupDetail(retired)))
630	698		}
631	699		}
	700	+	return s.startCandidate(ctx, app, revision, p, opID, before, journal)
	701	+	}
	702	+
	703	+	// deployOverlap creates the replacement of an overlap-eligible service while
	704	+	// the superseded container keeps running and stays routed: no inhibition and
	705	+	// no retirement. Container names carry the operation, so the two generations
	706	+	// never collide, and their loopback binds are ephemeral.
	707	+	//
	708	+	// The only withdrawal it can perform is the retry of one already pending for
	709	+	// this service: an earlier withdrawal that could not be proven leaves the
	710	+	// service marked, and a marked service would be withdrawn again by the
	711	+	// traffic refresh that follows the switch, erasing the binds the switch just
	712	+	// published. The retry runs before the candidate exists, so a failure aborts
	713	+	// the deploy untouched.
	714	+	//
	715	+	// It returns the ready candidate with its binds, after durably recording the
	716	+	// retirement-pending marker of the superseded container; on any failure the
	717	+	// candidate is removed and the superseded container is left exactly as it was.
	718	+	// Retiring the superseded container is the caller's job, once the candidate is
	719	+	// published and routed (retireSuperseded).
	720	+	func (s *Service) deployOverlap(ctx context.Context, app, revision string, p pinnedService, opID, before string, beforeGrace time.Duration, journal candidateJournal) ServiceResult {
	721	+	if failed, ok := s.revalidateService(app, revision, p, before); !ok {
	722	+	return failed
	723	+	}
	724	+	if s.publication.inhibited(app, p.name) {
	725	+	if err := s.withdrawForRecovery(ctx, app, p.name); err != nil {
	726	+	return s.failResult(revision, before, "", err)
	727	+	}
	728	+	}
	729	+	result := s.startCandidate(ctx, app, revision, p, opID, before, journal)
	730	+	if result.Result != "deployed" {
	731	+	return result
	732	+	}
	733	+	// The marker is durable before anything is published: whatever happens
	734	+	// next, the superseded container is retired by the periodic
	735	+	// reconciliation instead of being forgotten, and recovery never revives it.
	736	+	if err := s.inhibitRetirement(ctx, app, p.name, before, opID, beforeGrace); err != nil {
	737	+	result.Result = "failed"
	738	+	result.Error = err.Error()
	739	+	result.CleanupWarnings = append(result.CleanupWarnings, s.retireCandidate(ctx, app, p.name, result.After)...)
	740	+	return result
	741	+	}
	742	+	result.SupersededRunning = true
	743	+	return result
	744	+	}
	745	+
	746	+	// revalidateService re-resolves the binds and devices of a service against
	747	+	// the current policy before any workload mutation. It returns the failed
	748	+	// result and false when either was revoked since preflight.
	749	+	func (s *Service) revalidateService(app, revision string, p pinnedService, before string) (ServiceResult, bool) {
	750	+	if _, err := s.resolveServiceBinds(app, p.spec); err != nil {
	751	+	return s.failResult(revision, before, "", err), false
	752	+	}
	753	+	if _, err := s.resolveServiceDevices(app, p.spec); err != nil {
	754	+	return s.failResult(revision, before, "", err), false
	755	+	}
	756	+	return ServiceResult{}, true
	757	+	}
	758	+
	759	+	// startCandidate creates and starts the replacement generation and waits for
	760	+	// its readiness probe. A candidate that fails readiness is removed after its
	761	+	// redacted diagnostics were captured; whatever generation it supersedes is
	762	+	// not touched here.
	763	+	func (s *Service) startCandidate(ctx context.Context, app, revision string, p pinnedService, opID, before string, journal candidateJournal) ServiceResult {
632	764		candidate, err := s.createAndStart(ctx, app, revision, p, opID, journal)
633	765		if err != nil {
634	766		result := s.failResult(revision, before, "", err)

@@ -642,10 +774,10 @@ func (s *Service) deployService(ctx context.Context, app, revision string, p pin
642	774		created := candidate.Container
643	775		if err := s.waitServiceReady(ctx, app, created.ID, deploymentReadiness(p.spec), candidate.TCPBinds); err != nil {
644	776		// Capture redacted diagnostics while the failed replacement still
645		-	// exists, then remove only that candidate. The superseded container
646		-	// is already gone and is never recreated. Cleanup is best effort: a
647		-	// canceled context can leave the candidate behind, which is then
648		-	// reported as a leftover warning instead of a false success.
	777	+	// exists, then remove only that candidate. A superseded container
	778	+	// that was already retired is never recreated. Cleanup is best
	779	+	// effort: a canceled context can leave the candidate behind, which is
	780	+	// then reported as a leftover warning instead of a false success.
649	781		tail := s.redactDiagnostics(ctx, app, p, s.logTail(ctx, created.ID))
650	782		cleanup := s.retireCandidate(ctx, app, p.name, created.ID)
651	783		return ServiceResult{

@@ -667,16 +799,42 @@ func (s *Service) deployService(ctx context.Context, app, revision string, p pin
667	799		RestartUnsafe: singleWriterRequired(p.spec),
668	800		BackendBinds: candidate.TCPBinds,
669	801		UDPBackendBinds: candidate.UDPBinds,
	802	+	VerifiedStartedAt: candidate.StartedAt,
670	803		}
671	804		}
672	805
673	806		// singleWriterRequired reports services that must never have two generations
674	807		// running concurrently. Persistent volumes and any bind (especially a
675	808		// writable one) may be written by both, so they are treated identically.
	809	+	// Devices and UDP interfaces are also excluded from the overlap (see
	810	+	// overlapEligible).
676	811		func singleWriterRequired(spec domain.AppService) bool {
677	812		return len(spec.Volumes) > 0 \|\| len(spec.Binds) > 0
678	813		}
679	814
	815	+	// canOverlap reports whether a replacement may run beside its predecessor: both
	816	+	// the new spec and the predecessor's effective spec (as recorded in ACTIVE)
	817	+	// must be overlap-eligible. A predecessor that owned a volume, bind, device,
	818	+	// or UDP interface is a single writer or has pinned UDP sessions whatever the
	819	+	// replacement declares, so removing the resource from the spec does not make
	820	+	// the transition safe.
	821	+	func canOverlap(next, previous domain.AppService) bool {
	822	+	return overlapEligible(next) && overlapEligible(previous)
	823	+	}
	824	+
	825	+	// overlapEligible reports whether a service may be replaced with a short
	826	+	// overlap of the old and the new generation: it has no volume, bind, device,
	827	+	// or UDP interface. Volumes, binds, and devices are stateful or exclusive
	828	+	// resources two generations must not share. UDP sessions are pinned by the
	829	+	// relay to the backend they started on and outlive the switch for the
	830	+	// configured drain timeout, so they would keep reaching a backend that is
	831	+	// about to be stopped and whose loopback port may be recycled. TCP is fine:
	832	+	// established relays do not redial. Every other service keeps the sequential
	833	+	// replacement, so two generations of such a service never run together.
	834	+	func overlapEligible(spec domain.AppService) bool {
	835	+	return len(spec.Volumes) == 0 && len(spec.Binds) == 0 && len(spec.Devices) == 0 && len(spec.UDP) == 0
	836	+	}
	837	+
680	838		// refreshTraffic rebuilds the proxy host index and applies the full
681	839		// HTTP/L4 graph after activation. A failure is fatal to the caller:
682	840		// ACTIVE is published but the new service is not routable, so the

@@ -748,15 +906,19 @@ func (s *Service) publishService(ctx context.Context, app, revision string, p pi
748	906		if err := s.recordOwnership(ctx, app, p); err != nil {
749	907		return err
750	908		}
751		-	// The superseded generation is now safely replaced: clearing its
752		-	// inhibition is what makes this generation authoritative. A failed
753		-	// publication leaves the marker in place.
754		-	if result.Before != "" && result.Before != result.After {
755		-	if err := s.clearRecoveryInhibition(ctx, app, p.name, result.Before); err != nil {
756		-	return err
757		-	}
	909	+	return s.clearSupersededMarker(ctx, app, p.name, result)
	910	+	}
	911	+
	912	+	// clearSupersededMarker clears the recovery inhibition of the generation a
	913	+	// sequential replacement superseded: that clearing is what makes the new
	914	+	// generation authoritative, and a failed publication leaves the marker in
	915	+	// place. An overlap replacement is the exception: its superseded container
	916	+	// still runs, and its retirement-pending marker is what gets it retired.
	917	+	func (s *Service) clearSupersededMarker(ctx context.Context, app, service string, result ServiceResult) error {
	918	+	if result.Before == "" \|\| result.Before == result.After \|\| result.SupersededRunning {
	919	+	return nil
758	920		}
759		-	return nil
	921	+	return s.clearRecoveryInhibition(ctx, app, service, result.Before)
760	922		}
761	923
762	924		// recordOwnership stamps volume/secret/network ownership after publication.

@@ -863,14 +1025,31 @@ func (s *Service) failResult(revision, before, after string, err error) ServiceR
863	1025		// HTTP interface, no L4 interface, no volume, and no bind. An undeclared
864	1026		// probe on a stateful or L4 workload stays unchecked instead: an HTTP GET on
865	1027		// the published port proves nothing there, so Gordon never guesses one.
	1028	+	//
	1029	+	// A stateless raw TCP service (see implicitTCPProbe) that declares no check
	1030	+	// receives the implicit TCP probe, so the overlap switch never routes to a
	1031	+	// container whose port does not accept yet.
866	1032		func deploymentReadiness(spec domain.AppService) domain.AppService {
867		-	if !implicitHTTPProbe(spec) {
868		-	return spec
	1033	+	switch {
	1034	+	case implicitHTTPProbe(spec):
	1035	+	spec.Readiness.Type = domain.AppReadinessHTTP
	1036	+	case implicitTCPProbe(spec):
	1037	+	spec.Readiness.Type = domain.AppReadinessTCP
869	1038		}
870		-	spec.Readiness.Type = domain.AppReadinessHTTP
871	1039		return spec
872	1040		}
873	1041
	1042	+	// implicitTCPProbe reports whether a service that declares no readiness check
	1043	+	// receives the implicit TCP probe: exactly one TCP interface, no HTTP or UDP
	1044	+	// interface, no volume, bind, or device. Such a service is replaced with an
	1045	+	// overlap, and its single published loopback bind is the probe target.
	1046	+	func implicitTCPProbe(spec domain.AppService) bool {
	1047	+	if spec.Readiness.Type != "" && spec.Readiness.Type != domain.AppReadinessNone {
	1048	+	return false
	1049	+	}
	1050	+	return len(spec.TCP) == 1 && len(spec.HTTP) == 0 && overlapEligible(spec)
	1051	+	}
	1052	+
874	1053		// implicitHTTPProbe reports whether a service that declares no readiness
875	1054		// check receives the implicit HTTP probe when a deploy or restart starts it.
876	1055		// Recovery verification of an already running generation keeps the declared


@@ -20,12 +20,16 @@ const failLogTailLines = 50
20		const failLogTailTimeout = 10 * time.Second	20		const failLogTailTimeout = 10 * time.Second
21			21
22		// Deploy executes a preflighted revision: fail-fast across services in	22		// Deploy executes a preflighted revision: fail-fast across services in
23	-	// sorted name order. Every service is replaced sequentially — withdraw	23	+	// sorted name order. A service without volumes, binds, devices, or UDP interfaces is
			24	+	// replaced with a short overlap: its replacement starts and becomes ready
			25	+	// while the superseded container keeps serving, traffic switches to it, and
			26	+	// only then is the superseded container stopped, so the service is not
			27	+	// interrupted. Every other service is replaced sequentially — withdraw
24		// traffic, retire the superseded container, start the replacement and wait	28		// traffic, retire the superseded container, start the replacement and wait
25	-	// for its readiness probe — so a deployment may briefly interrupt the	29	+	// for its readiness probe — so a deployment may briefly interrupt it and two
26	-	// service, and two Gordon-managed generations of one service never run at	30	+	// Gordon-managed generations of a single-writer service never run at the same
27	-	// the same time. Volumes are never deleted: no RemoveVolume call, no	31	+	// time. Volumes are never deleted: no RemoveVolume call, no volume-deletion
28	-	// volume-deletion flags on container removal.	32	+	// flags on container removal.
29		//	33		//
30		// It is the synchronous composition of the two engine phases: StartDeploy	34		// It is the synchronous composition of the two engine phases: StartDeploy
31		// claims the key and journals the operation, then ExecuteDeploy runs the	35		// claims the key and journals the operation, then ExecuteDeploy runs the

@@ -147,7 +151,7 @@ func (s Service) startDeployLocked(ctx context.Context, input DeployInput) (St
147			151
148		// ExecuteDeploy performs the second phase of a deploy: it reacquires the app	152		// ExecuteDeploy performs the second phase of a deploy: it reacquires the app
149		// lock, loads and verifies the claimed journal by app/op identity, runs the	153		// lock, loads and verifies the claimed journal by app/op identity, runs the
150	-	// image pull/pin and the sequential replacement, and records the terminal	154	+	// image pull/pin and the per-service replacement, and records the terminal
151		// outcome. A claim that is already terminal, or that does not match its	155		// outcome. A claim that is already terminal, or that does not match its
152		// identity, is never executed: it replays its stored outcome instead. An	156		// identity, is never executed: it replays its stored outcome instead. An
153		// operation that stops mid-execution (shutdown or cancellation) leaves its	157		// operation that stops mid-execution (shutdown or cancellation) leaves its

@@ -305,6 +309,24 @@ func (s *Service) claimRevision(ctx context.Context, claim DeployClaim, op domai
305		// step is recorded as succeeded only after the runtime effect, the ACTIVE	309		// step is recorded as succeeded only after the runtime effect, the ACTIVE
306		// publication, and the traffic apply all succeeded, so a journaled success	310		// publication, and the traffic apply all succeeded, so a journaled success
307		// is never ahead of what is routed.	311		// is never ahead of what is routed.
			312	+	//
			313	+	// An overlap-eligible service (see overlapEligible) is replaced without
			314	+	// withdrawing the superseded container: it keeps serving and is retired only
			315	+	// after the new generation is published and routed. Before the switch the
			316	+	// superseded container gets a durable retirement-pending marker carrying its
			317	+	// stop grace. That marker, not the operation journal, owns the cleanup: it
			318	+	// survives newer operations, and the periodic reconciliation retires the
			319	+	// container once its successor is verified and routed.
			320	+	//
			321	+	// If the publication or the traffic apply of an overlap step fails, the
			322	+	// superseded container is deliberately left running: the proxy may still
			323	+	// route to it, and ACTIVE may already name the new container, so stopping it
			324	+	// could create an outage the failed deploy cannot repair. The step is
			325	+	// journaled as failed with the new container as After and a cleanup warning
			326	+	// names the superseded one. If ACTIVE names the new container, the periodic
			327	+	// reconciliation verifies and republishes it and then retires the superseded
			328	+	// one through its marker; if ACTIVE never changed, it still names the
			329	+	// superseded container, which keeps serving, and the marker is dropped.
308		func (s *Service) runServiceStep(	330		func (s *Service) runServiceStep(
309		ctx context.Context,	331		ctx context.Context,
310		app, revision string,	332		app, revision string,

@@ -331,6 +353,10 @@ func (s *Service) runServiceStep(
331		Before: before, State: domain.AppStepPending,	353		Before: before, State: domain.AppStepPending,
332		}	354		}
333		var svcResult ServiceResult	355		var svcResult ServiceResult
			356	+	overlap := false
			357	+	// The grace of the generation being superseded, read before anything
			358	+	// publishes a record of its replacement.
			359	+	beforeGrace := activeStopGrace(active, p.name)
334		if current, ok := s.unchangedService(ctx, app, p, active); ok {	360		if current, ok := s.unchangedService(ctx, app, p, active); ok {
335		// Same image, spec, environment, and secret values already running:	361		// Same image, spec, environment, and secret values already running:
336		// keep the container.	362		// keep the container.

@@ -341,8 +367,11 @@ func (s *Service) runServiceStep(
341		RestartUnsafe: singleWriterRequired(p.spec),	367		RestartUnsafe: singleWriterRequired(p.spec),
342		}	368		}
343		step.Detail = domain.AppServiceUnchanged + ": already running " + p.spec.Image + " (" + p.digest + ")"	369		step.Detail = domain.AppServiceUnchanged + ": already running " + p.spec.Image + " (" + p.digest + ")"
			370	+	} else if before != "" && canOverlap(p.spec, active.Services[p.name].Spec) {
			371	+	overlap = true
			372	+	svcResult = s.deployOverlap(ctx, app, revision, p, op.Op, before, beforeGrace, s.journalCandidate(op, index+1))
344		} else {	373		} else {
345	-	svcResult = s.deployService(ctx, app, revision, p, op.Op, before, activeStopGrace(active, p.name), s.journalCandidate(op, index+1))	374	+	svcResult = s.deployService(ctx, app, revision, p, op.Op, before, beforeGrace, s.journalCandidate(op, index+1))
346		}	375		}
347		result.Services[p.name] = svcResult	376		result.Services[p.name] = svcResult
348		// A recorded candidate is authoritative while the step is unresolved: a	377		// A recorded candidate is authoritative while the step is unresolved: a

@@ -367,22 +396,22 @@ func (s *Service) runServiceStep(
367		return fail(fmt.Errorf("deployment: service %q failed at %s: %s: %w",	396		return fail(fmt.Errorf("deployment: service %q failed at %s: %s: %w",
368		p.name, stepID, svcResult.Error, domain.ErrAppStateConflict))	397		p.name, stepID, svcResult.Error, domain.ErrAppStateConflict))
369		}	398		}
370	-	// Publish the per-service effective state: the new binds become the	399	+	svcResult, err := s.publishAndRoute(ctx, app, revision, p, svcResult, op.Op)
371	-	// service's recorded backend before the proxy is repointed at them.	400	+	if err != nil {
372	-	if err := s.publishService(ctx, app, revision, p, svcResult, op.Op); err != nil {	401	+	if overlap {
373	-	svcResult.Result = "failed"	402	+	svcResult.CleanupWarnings = append(svcResult.CleanupWarnings, CleanupWarning{
374	-	svcResult.Error = err.Error()	403	+	Service: p.name, Leftover: before,
			404	+	Detail: "superseded container kept running: the new generation was not confirmed published and routed",
			405	+	})
			406	+	}
375		result.Services[p.name] = svcResult	407		result.Services[p.name] = svcResult
376		return fail(err)	408		return fail(err)
377		}	409		}
378	-	// Rebuild and publish the traffic graph. ACTIVE already names the new	410	+	// The generation is verified and routed: the periodic pass must not
379	-	// container, but the operation is a failure until routing accepted it,	411	+	// mistake its first observation for an unseen execution and withdraw it.
380	-	// so a rejected graph must never be journaled as success.	412	+	s.recordVerifiedExecution(app, p.name, svcResult)
381	-	if err := s.refreshTraffic(ctx, app); err != nil {	413	+	if overlap {
382	-	svcResult.Result = "failed"	414	+	svcResult = s.retireSuperseded(ctx, app, p.name, before, beforeGrace, svcResult)
383	-	svcResult.Error = err.Error()
384	-	result.Services[p.name] = svcResult
385	-	return fail(err)
386		}	415		}
387		result.Services[p.name] = svcResult	416		result.Services[p.name] = svcResult
388		step.State = domain.AppStepSucceeded	417		step.State = domain.AppStepSucceeded

@@ -393,6 +422,45 @@ func (s *Service) runServiceStep(
393		return nil	422		return nil
394		}	423		}
395			424
			425	+	// publishAndRoute publishes the per-service effective state, so the new
			426	+	// binds become the service's recorded backend, then rebuilds the traffic
			427	+	// graph so the proxy is repointed at them. ACTIVE already names the new
			428	+	// container when the graph is applied, but the operation is a failure until
			429	+	// routing accepted it, so a rejected graph is returned as a failed result and
			430	+	// never journaled as success.
			431	+	func (s *Service) publishAndRoute(ctx context.Context, app, revision string, p pinnedService, svcResult ServiceResult, opID string) (ServiceResult, error) {
			432	+	if err := s.publishService(ctx, app, revision, p, svcResult, opID); err != nil {
			433	+	svcResult.Result = "failed"
			434	+	svcResult.Error = err.Error()
			435	+	return svcResult, err
			436	+	}
			437	+	if err := s.refreshTraffic(ctx, app); err != nil {
			438	+	svcResult.Result = "failed"
			439	+	svcResult.Error = err.Error()
			440	+	return svcResult, err
			441	+	}
			442	+	return svcResult, nil
			443	+	}
			444	+
			445	+	// retireSuperseded stops the container an overlap replacement superseded,
			446	+	// once the new generation is published and routed. It uses the stop grace of
			447	+	// the generation being stopped: the runtime sends SIGTERM, then kills after
			448	+	// the grace. Backend claims are released and the retirement-pending marker is
			449	+	// cleared by retireContainer once the container is confirmed gone. A
			450	+	// retirement that fails is not a deploy failure — the new generation already
			451	+	// serves — but a cleanup warning on the service; the marker stays, so the
			452	+	// periodic reconciliation retries it.
			453	+	func (s *Service) retireSuperseded(ctx context.Context, app, service, before string, grace time.Duration, result ServiceResult) ServiceResult {
			454	+	retired := s.retireContainer(ctx, app, retireOptions{Service: service, Grace: grace, ClearInhibition: true, RetryOnLeak: true}, before)
			455	+	result.CleanupWarnings = append(result.CleanupWarnings, retired.Warnings...)
			456	+	if !retired.Gone && len(retired.Warnings) == 0 {
			457	+	result.CleanupWarnings = append(result.CleanupWarnings, CleanupWarning{
			458	+	Service: service, Leftover: before, Detail: "superseded container not confirmed gone",
			459	+	})
			460	+	}
			461	+	return result
			462	+	}
			463	+
396		// unchangedService reports whether the service already runs exactly what p	464		// unchangedService reports whether the service already runs exactly what p
397		// would create, in a healthy published state: the same digest, service spec,	465		// would create, in a healthy published state: the same digest, service spec,
398		// app environment, and shared networks, in a running container that is not	466		// app environment, and shared networks, in a running container that is not

@@ -579,7 +647,10 @@ func (s Service) journalCandidate(op domain.AppOperation, index int) candidate
579			647
580		// deployService replaces one service sequentially and returns its terminal	648		// deployService replaces one service sequentially and returns its terminal
581		// result. before is the ACTIVE container being superseded (empty on a first	649		// result. before is the ACTIVE container being superseded (empty on a first
582	-	// deployment); beforeGrace is its effective stop grace.	650	+	// deployment); beforeGrace is its effective stop grace. It serves every
			651	+	// service that is not overlap-eligible, a first deployment, and the
			652	+	// rebuild of a missing generation by start and restart; deployOverlap is the
			653	+	// overlapping alternative used by a deploy.
583		//	654		//
584		// The order is fixed: withdraw from traffic and confirm it, durably inhibit	655		// The order is fixed: withdraw from traffic and confirm it, durably inhibit
585		// a single-writer generation, retire the superseded container and confirm	656		// a single-writer generation, retire the superseded container and confirm

@@ -591,11 +662,8 @@ func (s *Service) deployService(ctx context.Context, app, revision string, p pin
591		// Revalidate the current authorization and source before withdrawing or	662		// Revalidate the current authorization and source before withdrawing or
592		// retiring the serving generation. A reload between preflight and this	663		// retiring the serving generation. A reload between preflight and this
593		// service step must fail without mutating the existing workload.	664		// service step must fail without mutating the existing workload.
594	-	if _, err := s.resolveServiceBinds(app, p.spec); err != nil {	665	+	if failed, ok := s.revalidateService(app, revision, p, before); !ok {
595	-	return s.failResult(revision, before, "", err)	666	+	return failed
596	-	}
597	-	if _, err := s.resolveServiceDevices(app, p.spec); err != nil {
598	-	return s.failResult(revision, before, "", err)
599		}	667		}
600		// Withdraw the service from traffic first and confirm it: a withdrawal	668		// Withdraw the service from traffic first and confirm it: a withdrawal
601		// that cannot be applied must block every container mutation, so no	669		// that cannot be applied must block every container mutation, so no

@@ -629,6 +697,70 @@ func (s *Service) deployService(ctx context.Context, app, revision string, p pin
629		return s.failResult(revision, before, "", fmt.Errorf("deployment: retire superseded container %s: %s", before, cleanupDetail(retired)))	697		return s.failResult(revision, before, "", fmt.Errorf("deployment: retire superseded container %s: %s", before, cleanupDetail(retired)))
630		}	698		}
631		}	699		}
			700	+	return s.startCandidate(ctx, app, revision, p, opID, before, journal)
			701	+	}
			702	+
			703	+	// deployOverlap creates the replacement of an overlap-eligible service while
			704	+	// the superseded container keeps running and stays routed: no inhibition and
			705	+	// no retirement. Container names carry the operation, so the two generations
			706	+	// never collide, and their loopback binds are ephemeral.
			707	+	//
			708	+	// The only withdrawal it can perform is the retry of one already pending for
			709	+	// this service: an earlier withdrawal that could not be proven leaves the
			710	+	// service marked, and a marked service would be withdrawn again by the
			711	+	// traffic refresh that follows the switch, erasing the binds the switch just
			712	+	// published. The retry runs before the candidate exists, so a failure aborts
			713	+	// the deploy untouched.
			714	+	//
			715	+	// It returns the ready candidate with its binds, after durably recording the
			716	+	// retirement-pending marker of the superseded container; on any failure the
			717	+	// candidate is removed and the superseded container is left exactly as it was.
			718	+	// Retiring the superseded container is the caller's job, once the candidate is
			719	+	// published and routed (retireSuperseded).
			720	+	func (s *Service) deployOverlap(ctx context.Context, app, revision string, p pinnedService, opID, before string, beforeGrace time.Duration, journal candidateJournal) ServiceResult {
			721	+	if failed, ok := s.revalidateService(app, revision, p, before); !ok {
			722	+	return failed
			723	+	}
			724	+	if s.publication.inhibited(app, p.name) {
			725	+	if err := s.withdrawForRecovery(ctx, app, p.name); err != nil {
			726	+	return s.failResult(revision, before, "", err)
			727	+	}
			728	+	}
			729	+	result := s.startCandidate(ctx, app, revision, p, opID, before, journal)
			730	+	if result.Result != "deployed" {
			731	+	return result
			732	+	}
			733	+	// The marker is durable before anything is published: whatever happens
			734	+	// next, the superseded container is retired by the periodic
			735	+	// reconciliation instead of being forgotten, and recovery never revives it.
			736	+	if err := s.inhibitRetirement(ctx, app, p.name, before, opID, beforeGrace); err != nil {
			737	+	result.Result = "failed"
			738	+	result.Error = err.Error()
			739	+	result.CleanupWarnings = append(result.CleanupWarnings, s.retireCandidate(ctx, app, p.name, result.After)...)
			740	+	return result
			741	+	}
			742	+	result.SupersededRunning = true
			743	+	return result
			744	+	}
			745	+
			746	+	// revalidateService re-resolves the binds and devices of a service against
			747	+	// the current policy before any workload mutation. It returns the failed
			748	+	// result and false when either was revoked since preflight.
			749	+	func (s *Service) revalidateService(app, revision string, p pinnedService, before string) (ServiceResult, bool) {
			750	+	if _, err := s.resolveServiceBinds(app, p.spec); err != nil {
			751	+	return s.failResult(revision, before, "", err), false
			752	+	}
			753	+	if _, err := s.resolveServiceDevices(app, p.spec); err != nil {
			754	+	return s.failResult(revision, before, "", err), false
			755	+	}
			756	+	return ServiceResult{}, true
			757	+	}
			758	+
			759	+	// startCandidate creates and starts the replacement generation and waits for
			760	+	// its readiness probe. A candidate that fails readiness is removed after its
			761	+	// redacted diagnostics were captured; whatever generation it supersedes is
			762	+	// not touched here.
			763	+	func (s *Service) startCandidate(ctx context.Context, app, revision string, p pinnedService, opID, before string, journal candidateJournal) ServiceResult {
632		candidate, err := s.createAndStart(ctx, app, revision, p, opID, journal)	764		candidate, err := s.createAndStart(ctx, app, revision, p, opID, journal)
633		if err != nil {	765		if err != nil {
634		result := s.failResult(revision, before, "", err)	766		result := s.failResult(revision, before, "", err)

@@ -642,10 +774,10 @@ func (s *Service) deployService(ctx context.Context, app, revision string, p pin
642		created := candidate.Container	774		created := candidate.Container
643		if err := s.waitServiceReady(ctx, app, created.ID, deploymentReadiness(p.spec), candidate.TCPBinds); err != nil {	775		if err := s.waitServiceReady(ctx, app, created.ID, deploymentReadiness(p.spec), candidate.TCPBinds); err != nil {
644		// Capture redacted diagnostics while the failed replacement still	776		// Capture redacted diagnostics while the failed replacement still
645	-	// exists, then remove only that candidate. The superseded container	777	+	// exists, then remove only that candidate. A superseded container
646	-	// is already gone and is never recreated. Cleanup is best effort: a	778	+	// that was already retired is never recreated. Cleanup is best
647	-	// canceled context can leave the candidate behind, which is then	779	+	// effort: a canceled context can leave the candidate behind, which is
648	-	// reported as a leftover warning instead of a false success.	780	+	// then reported as a leftover warning instead of a false success.
649		tail := s.redactDiagnostics(ctx, app, p, s.logTail(ctx, created.ID))	781		tail := s.redactDiagnostics(ctx, app, p, s.logTail(ctx, created.ID))
650		cleanup := s.retireCandidate(ctx, app, p.name, created.ID)	782		cleanup := s.retireCandidate(ctx, app, p.name, created.ID)
651		return ServiceResult{	783		return ServiceResult{

@@ -667,16 +799,42 @@ func (s *Service) deployService(ctx context.Context, app, revision string, p pin
667		RestartUnsafe: singleWriterRequired(p.spec),	799		RestartUnsafe: singleWriterRequired(p.spec),
668		BackendBinds: candidate.TCPBinds,	800		BackendBinds: candidate.TCPBinds,
669		UDPBackendBinds: candidate.UDPBinds,	801		UDPBackendBinds: candidate.UDPBinds,
			802	+	VerifiedStartedAt: candidate.StartedAt,
670		}	803		}
671		}	804		}
672			805
673		// singleWriterRequired reports services that must never have two generations	806		// singleWriterRequired reports services that must never have two generations
674		// running concurrently. Persistent volumes and any bind (especially a	807		// running concurrently. Persistent volumes and any bind (especially a
675		// writable one) may be written by both, so they are treated identically.	808		// writable one) may be written by both, so they are treated identically.
			809	+	// Devices and UDP interfaces are also excluded from the overlap (see
			810	+	// overlapEligible).
676		func singleWriterRequired(spec domain.AppService) bool {	811		func singleWriterRequired(spec domain.AppService) bool {
677		return len(spec.Volumes) > 0 \|\| len(spec.Binds) > 0	812		return len(spec.Volumes) > 0 \|\| len(spec.Binds) > 0
678		}	813		}
679			814
			815	+	// canOverlap reports whether a replacement may run beside its predecessor: both
			816	+	// the new spec and the predecessor's effective spec (as recorded in ACTIVE)
			817	+	// must be overlap-eligible. A predecessor that owned a volume, bind, device,
			818	+	// or UDP interface is a single writer or has pinned UDP sessions whatever the
			819	+	// replacement declares, so removing the resource from the spec does not make
			820	+	// the transition safe.
			821	+	func canOverlap(next, previous domain.AppService) bool {
			822	+	return overlapEligible(next) && overlapEligible(previous)
			823	+	}
			824	+
			825	+	// overlapEligible reports whether a service may be replaced with a short
			826	+	// overlap of the old and the new generation: it has no volume, bind, device,
			827	+	// or UDP interface. Volumes, binds, and devices are stateful or exclusive
			828	+	// resources two generations must not share. UDP sessions are pinned by the
			829	+	// relay to the backend they started on and outlive the switch for the
			830	+	// configured drain timeout, so they would keep reaching a backend that is
			831	+	// about to be stopped and whose loopback port may be recycled. TCP is fine:
			832	+	// established relays do not redial. Every other service keeps the sequential
			833	+	// replacement, so two generations of such a service never run together.
			834	+	func overlapEligible(spec domain.AppService) bool {
			835	+	return len(spec.Volumes) == 0 && len(spec.Binds) == 0 && len(spec.Devices) == 0 && len(spec.UDP) == 0
			836	+	}
			837	+
680		// refreshTraffic rebuilds the proxy host index and applies the full	838		// refreshTraffic rebuilds the proxy host index and applies the full
681		// HTTP/L4 graph after activation. A failure is fatal to the caller:	839		// HTTP/L4 graph after activation. A failure is fatal to the caller:
682		// ACTIVE is published but the new service is not routable, so the	840		// ACTIVE is published but the new service is not routable, so the

@@ -748,15 +906,19 @@ func (s *Service) publishService(ctx context.Context, app, revision string, p pi
748		if err := s.recordOwnership(ctx, app, p); err != nil {	906		if err := s.recordOwnership(ctx, app, p); err != nil {
749		return err	907		return err
750		}	908		}
751	-	// The superseded generation is now safely replaced: clearing its	909	+	return s.clearSupersededMarker(ctx, app, p.name, result)
752	-	// inhibition is what makes this generation authoritative. A failed	910	+	}
753	-	// publication leaves the marker in place.	911	+
754	-	if result.Before != "" && result.Before != result.After {	912	+	// clearSupersededMarker clears the recovery inhibition of the generation a
755	-	if err := s.clearRecoveryInhibition(ctx, app, p.name, result.Before); err != nil {	913	+	// sequential replacement superseded: that clearing is what makes the new
756	-	return err	914	+	// generation authoritative, and a failed publication leaves the marker in
757	-	}	915	+	// place. An overlap replacement is the exception: its superseded container
			916	+	// still runs, and its retirement-pending marker is what gets it retired.
			917	+	func (s *Service) clearSupersededMarker(ctx context.Context, app, service string, result ServiceResult) error {
			918	+	if result.Before == "" \|\| result.Before == result.After \|\| result.SupersededRunning {
			919	+	return nil
758		}	920		}
759	-	return nil	921	+	return s.clearRecoveryInhibition(ctx, app, service, result.Before)
760		}	922		}
761			923
762		// recordOwnership stamps volume/secret/network ownership after publication.	924		// recordOwnership stamps volume/secret/network ownership after publication.

@@ -863,14 +1025,31 @@ func (s *Service) failResult(revision, before, after string, err error) ServiceR
863		// HTTP interface, no L4 interface, no volume, and no bind. An undeclared	1025		// HTTP interface, no L4 interface, no volume, and no bind. An undeclared
864		// probe on a stateful or L4 workload stays unchecked instead: an HTTP GET on	1026		// probe on a stateful or L4 workload stays unchecked instead: an HTTP GET on
865		// the published port proves nothing there, so Gordon never guesses one.	1027		// the published port proves nothing there, so Gordon never guesses one.
			1028	+	//
			1029	+	// A stateless raw TCP service (see implicitTCPProbe) that declares no check
			1030	+	// receives the implicit TCP probe, so the overlap switch never routes to a
			1031	+	// container whose port does not accept yet.
866		func deploymentReadiness(spec domain.AppService) domain.AppService {	1032		func deploymentReadiness(spec domain.AppService) domain.AppService {
867	-	if !implicitHTTPProbe(spec) {	1033	+	switch {
868	-	return spec	1034	+	case implicitHTTPProbe(spec):
			1035	+	spec.Readiness.Type = domain.AppReadinessHTTP
			1036	+	case implicitTCPProbe(spec):
			1037	+	spec.Readiness.Type = domain.AppReadinessTCP
869		}	1038		}
870	-	spec.Readiness.Type = domain.AppReadinessHTTP
871		return spec	1039		return spec
872		}	1040		}
873			1041
			1042	+	// implicitTCPProbe reports whether a service that declares no readiness check
			1043	+	// receives the implicit TCP probe: exactly one TCP interface, no HTTP or UDP
			1044	+	// interface, no volume, bind, or device. Such a service is replaced with an
			1045	+	// overlap, and its single published loopback bind is the probe target.
			1046	+	func implicitTCPProbe(spec domain.AppService) bool {
			1047	+	if spec.Readiness.Type != "" && spec.Readiness.Type != domain.AppReadinessNone {
			1048	+	return false
			1049	+	}
			1050	+	return len(spec.TCP) == 1 && len(spec.HTTP) == 0 && overlapEligible(spec)
			1051	+	}
			1052	+
874		// implicitHTTPProbe reports whether a service that declares no readiness	1053		// implicitHTTPProbe reports whether a service that declares no readiness
875		// check receives the implicit HTTP probe when a deploy or restart starts it.	1054		// check receives the implicit HTTP probe when a deploy or restart starts it.
876		// Recovery verification of an already running generation keeps the declared	1055		// Recovery verification of an already running generation keeps the declared

A internal/usecase/deployment/execution_start_internal_test.go +8 -0

@@ -0,0 +1,8 @@
	1	+	package deployment
	2	+
	3	+	import "time"
	4	+
	5	+	// executionStartedAt is the start time the fakes of the internal tests report
	6	+	// for a freshly started container: deploy and restart observe it right after
	7	+	// the start.
	8	+	var executionStartedAt = time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)

@@ -0,0 +1,8 @@
	1	+	package deployment
	2	+
	3	+	import "time"
	4	+
	5	+	// executionStartedAt is the start time the fakes of the internal tests report
	6	+	// for a freshly started container: deploy and restart observe it right after
	7	+	// the start.
	8	+	var executionStartedAt = time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)

M internal/usecase/deployment/interrupted.go +34 -6


@@ -155,11 +155,19 @@ func (s Service) convergeInterruptedSteps(ctx context.Context, app string, op
155	155		// in place, keeping a failed convergence eligible for retry.
156	156		//
157	157		// Once the candidate is converged, the inhibition the replacement wrote for
158		-	// the superseded container is cleared. A step carrying both Before and After
159		-	// records that the superseded container was confirmed gone before the
160		-	// candidate was created (deployService retires it first), so the marker
161		-	// protects nothing and would otherwise refuse the boot/start/restart rebuild
162		-	// from ACTIVE.
	158	+	// the superseded container is cleared. A sequential replacement confirmed the
	159	+	// superseded container gone before the candidate was created (deployService
	160	+	// retires it first), so its marker protects nothing and would otherwise
	161	+	// refuse the boot/start/restart rebuild from ACTIVE. An overlap replacement
	162	+	// leaves its superseded container running: when the candidate is published,
	163	+	// its retirement-pending marker stays, and the periodic reconciliation
	164	+	// retires that container once the candidate is verified and routed. When the
	165	+	// candidate was never published the superseded container still serves, so
	166	+	// the marker is dropped and the container is kept.
	167	+	//
	168	+	// Reconciliation never touches traffic: the persisted binds of a published
	169	+	// candidate are unverified here, and the verified start/recovery path
	170	+	// publishes them.
163	171		func (s Service) convergeCandidate(ctx context.Context, app string, active domain.AppActive, step domain.AppOperationStep, terminal bool, log zerowrap.Logger) ([]CleanupWarning, error) {
164	172		candidate := step.After
165	173		if publishedContainer(active, candidate) {

@@ -193,7 +201,7 @@ func (s *Service) convergeCandidate(ctx context.Context, app string, active doma
193	201		// writer is preserved because the superseded container was proven gone
194	202		// before the candidate existed.
195	203		if step.Service != "" && step.Before != "" && step.Before != candidate {
196		-	if err := s.clearRecoveryInhibition(ctx, app, step.Service, step.Before); err != nil {
	204	+	if err := s.clearSupersededInhibition(ctx, app, step.Service, step.Before, publishedContainer(active, candidate)); err != nil {
197	205		log.Warn().Err(err).Str("app", app).Str("container", step.Before).Msg("deployment: clear stale replacement inhibition")
198	206		return []CleanupWarning{{Service: step.Service, Leftover: step.Before, Detail: "clear recovery inhibition: " + err.Error()}}, nil
199	207		}

@@ -204,6 +212,26 @@ func (s *Service) convergeCandidate(ctx context.Context, app string, active doma
204	212		return nil, nil
205	213		}
206	214
	215	+	// clearSupersededInhibition drops the inhibition the interrupted replacement
	216	+	// recorded for the superseded container. A retirement-pending marker of a
	217	+	// published candidate is kept: the superseded container of an overlap
	218	+	// replacement may still be running, and that marker is what makes the
	219	+	// periodic reconciliation retire it. Every other marker is stale.
	220	+	func (s *Service) clearSupersededInhibition(ctx context.Context, app, service, before string, published bool) error {
	221	+	if published {
	222	+	inhibitions, err := s.deps.State.LoadRecoveryInhibitions(ctx, app)
	223	+	if err != nil {
	224	+	return fmt.Errorf("deployment: load recovery inhibitions: %w", err)
	225	+	}
	226	+	for _, inhibition := range inhibitions {
	227	+	if inhibition.Service == service && inhibition.ContainerID == before && inhibition.Reason == domain.AppInhibitRetirementPending {
	228	+	return nil
	229	+	}
	230	+	}
	231	+	}
	232	+	return s.clearRecoveryInhibition(ctx, app, service, before)
	233	+	}
	234	+
207	235		// interruptedOutcome classifies an interrupted operation: a failure, unless
208	236		// every step had already succeeded.
209	237		func interruptedOutcome(op domain.AppOperation) string {


@@ -155,11 +155,19 @@ func (s Service) convergeInterruptedSteps(ctx context.Context, app string, op
155		// in place, keeping a failed convergence eligible for retry.	155		// in place, keeping a failed convergence eligible for retry.
156		//	156		//
157		// Once the candidate is converged, the inhibition the replacement wrote for	157		// Once the candidate is converged, the inhibition the replacement wrote for
158	-	// the superseded container is cleared. A step carrying both Before and After	158	+	// the superseded container is cleared. A sequential replacement confirmed the
159	-	// records that the superseded container was confirmed gone before the	159	+	// superseded container gone before the candidate was created (deployService
160	-	// candidate was created (deployService retires it first), so the marker	160	+	// retires it first), so its marker protects nothing and would otherwise
161	-	// protects nothing and would otherwise refuse the boot/start/restart rebuild	161	+	// refuse the boot/start/restart rebuild from ACTIVE. An overlap replacement
162	-	// from ACTIVE.	162	+	// leaves its superseded container running: when the candidate is published,
			163	+	// its retirement-pending marker stays, and the periodic reconciliation
			164	+	// retires that container once the candidate is verified and routed. When the
			165	+	// candidate was never published the superseded container still serves, so
			166	+	// the marker is dropped and the container is kept.
			167	+	//
			168	+	// Reconciliation never touches traffic: the persisted binds of a published
			169	+	// candidate are unverified here, and the verified start/recovery path
			170	+	// publishes them.
163		func (s Service) convergeCandidate(ctx context.Context, app string, active domain.AppActive, step domain.AppOperationStep, terminal bool, log zerowrap.Logger) ([]CleanupWarning, error) {	171		func (s Service) convergeCandidate(ctx context.Context, app string, active domain.AppActive, step domain.AppOperationStep, terminal bool, log zerowrap.Logger) ([]CleanupWarning, error) {
164		candidate := step.After	172		candidate := step.After
165		if publishedContainer(active, candidate) {	173		if publishedContainer(active, candidate) {

@@ -193,7 +201,7 @@ func (s *Service) convergeCandidate(ctx context.Context, app string, active doma
193		// writer is preserved because the superseded container was proven gone	201		// writer is preserved because the superseded container was proven gone
194		// before the candidate existed.	202		// before the candidate existed.
195		if step.Service != "" && step.Before != "" && step.Before != candidate {	203		if step.Service != "" && step.Before != "" && step.Before != candidate {
196	-	if err := s.clearRecoveryInhibition(ctx, app, step.Service, step.Before); err != nil {	204	+	if err := s.clearSupersededInhibition(ctx, app, step.Service, step.Before, publishedContainer(active, candidate)); err != nil {
197		log.Warn().Err(err).Str("app", app).Str("container", step.Before).Msg("deployment: clear stale replacement inhibition")	205		log.Warn().Err(err).Str("app", app).Str("container", step.Before).Msg("deployment: clear stale replacement inhibition")
198		return []CleanupWarning{{Service: step.Service, Leftover: step.Before, Detail: "clear recovery inhibition: " + err.Error()}}, nil	206		return []CleanupWarning{{Service: step.Service, Leftover: step.Before, Detail: "clear recovery inhibition: " + err.Error()}}, nil
199		}	207		}

@@ -204,6 +212,26 @@ func (s *Service) convergeCandidate(ctx context.Context, app string, active doma
204		return nil, nil	212		return nil, nil
205		}	213		}
206			214
			215	+	// clearSupersededInhibition drops the inhibition the interrupted replacement
			216	+	// recorded for the superseded container. A retirement-pending marker of a
			217	+	// published candidate is kept: the superseded container of an overlap
			218	+	// replacement may still be running, and that marker is what makes the
			219	+	// periodic reconciliation retire it. Every other marker is stale.
			220	+	func (s *Service) clearSupersededInhibition(ctx context.Context, app, service, before string, published bool) error {
			221	+	if published {
			222	+	inhibitions, err := s.deps.State.LoadRecoveryInhibitions(ctx, app)
			223	+	if err != nil {
			224	+	return fmt.Errorf("deployment: load recovery inhibitions: %w", err)
			225	+	}
			226	+	for _, inhibition := range inhibitions {
			227	+	if inhibition.Service == service && inhibition.ContainerID == before && inhibition.Reason == domain.AppInhibitRetirementPending {
			228	+	return nil
			229	+	}
			230	+	}
			231	+	}
			232	+	return s.clearRecoveryInhibition(ctx, app, service, before)
			233	+	}
			234	+
207		// interruptedOutcome classifies an interrupted operation: a failure, unless	235		// interruptedOutcome classifies an interrupted operation: a failure, unless
208		// every step had already succeeded.	236		// every step had already succeeded.
209		func interruptedOutcome(op domain.AppOperation) string {	237		func interruptedOutcome(op domain.AppOperation) string {

M internal/usecase/deployment/lifecycle.go +99 -3


@@ -86,8 +86,22 @@ func (s Service) stopLocked(ctx context.Context, app, opID string) (LifecycleR
86	86		}
87	87		result.Services[name] = ServiceResult{Result: "deployed", Before: container, After: "", CleanupWarnings: warnings}
88	88		}
	89	+	// A superseded predecessor an overlap replacement left running is not in
	90	+	// ACTIVE, so the loop above never saw it: a stopped app must not keep it
	91	+	// running. One that cannot be retired keeps its marker and fails the stop.
	92	+	pendingFailed, pendingErr := s.retirePendingForStop(ctx, app, result)
	93	+	failures = append(failures, pendingFailed...)
89	94		op.Outcome = ComputeOutcome(result.Services)
90		-	op.Warnings = journalWarnings(collectCleanupWarnings(result.Services))
	95	+	if pendingErr != nil {
	96	+	// An app-level cleanup failure belongs to no service, so the
	97	+	// per-service results alone would journal a success and a replay of
	98	+	// this key would read it. It is a failed step and a failed outcome.
	99	+	op.Steps = append(op.Steps, domain.AppOperationStep{
	100	+	ID: "pending.retire", State: domain.AppStepFailed, Error: pendingErr.Error(),
	101	+	})
	102	+	op.Outcome = domain.AppOutcomeFailed
	103	+	}
	104	+	op.Warnings = journalWarnings(append(collectCleanupWarnings(result.Services), result.Warnings...))
91	105		if err := s.deps.State.SaveOperation(ctx, op); err != nil {
92	106		log.Warn().Err(err).Msg("deployment: failed to record stop outcome")
93	107		}

@@ -198,6 +212,10 @@ func (s Service) startLocked(ctx context.Context, app, opID string) (Lifecycle
198	212		s.failTrafficPublication(ctx, &op, err)
199	213		return result, err
200	214		}
	215	+	// Routing accepted the verified generations: record their executions.
	216	+	for name, svcResult := range result.Services {
	217	+	s.recordVerifiedExecution(app, name, svcResult)
	218	+	}
201	219		return result, nil
202	220		}
203	221

@@ -273,6 +291,11 @@ func (s *Service) restartLocked(ctx context.Context, app, service, opID string)
273	291		s.failTrafficPublication(ctx, &op, err)
274	292		return result, err
275	293		}
	294	+	// Routing accepted the restarted generations: their executions are
	295	+	// verified. The app lock is held, so no periodic pass ran in between.
	296	+	for name, svcResult := range result.Services {
	297	+	s.recordVerifiedExecution(app, name, svcResult)
	298	+	}
276	299		if len(failures) > 0 {
277	300		return result, fmt.Errorf("deployment: restart failed for %s: %w", strings.Join(failures, ", "), domain.ErrAppStateConflict)
278	301		}

@@ -347,6 +370,8 @@ func (s *Service) restartOneService(ctx context.Context, app, opID, name string,
347	370		s.clearServiceBinds(ctx, app, name)
348	371		return fail(err.Error())
349	372		}
	373	+	// The restarted execution is observed before readiness runs.
	374	+	startedAt := s.executionStart(ctx, eff.Container)
350	375		// Ephemeral loopback binds are not guaranteed stable across a runtime
351	376		// restart: re-inspect before probing so the proxy never dials a stale
352	377		// bind.

@@ -361,7 +386,7 @@ func (s *Service) restartOneService(ctx context.Context, app, opID, name string,
361	386		}
362	387		step.State = domain.AppStepSucceeded
363	388		step.After = eff.Container
364		-	return ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds}
	389	+	return ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds, VerifiedStartedAt: startedAt}
365	390		}
366	391
367	392		// Remove withdraws workloads by exact container ID; volumes, secrets, and

@@ -512,9 +537,75 @@ func (s *Service) removeServiceContainers(ctx context.Context, app, opID string,
512	537		step.State = domain.AppStepSucceeded
513	538		steps = append(steps, step)
514	539		}
	540	+	if err := s.retireAllPending(ctx, app); err != nil {
	541	+	return nil, err
	542	+	}
515	543		return steps, nil
516	544		}
517	545
	546	+	// retireAllPending retires every superseded container an overlap replacement
	547	+	// left running beside its successor. Ending the incarnation deletes the
	548	+	// retirement markers with the rest of the app state, so a container that is
	549	+	// not confirmed gone here would be orphaned and untracked: a failure keeps the
	550	+	// markers and fails the caller. Every marker is attempted, and a marker whose
	551	+	// claims could not be released is a failure too, so it is retried instead of
	552	+	// leaking its claims.
	553	+	func (s *Service) retireAllPending(ctx context.Context, app string) error {
	554	+	pending, err := s.pendingRetirements(ctx, app)
	555	+	if err != nil {
	556	+	return err
	557	+	}
	558	+	var failures []error
	559	+	for _, marker := range pending {
	560	+	retired := s.retireContainer(ctx, app, retireOptions{
	561	+	Service: marker.Service, Grace: marker.StopGrace, ClearInhibition: true, RetryOnLeak: true,
	562	+	}, marker.ContainerID)
	563	+	if !retired.Settled {
	564	+	failures = append(failures, fmt.Errorf("deployment: remove %q/%q: superseded container %s not retired: %s", app, marker.Service, marker.ContainerID, cleanupDetail(retired)))
	565	+	}
	566	+	}
	567	+	return errors.Join(failures...)
	568	+	}
	569	+
	570	+	// pendingRetirements loads the retirement-pending markers of one app.
	571	+	func (s *Service) pendingRetirements(ctx context.Context, app string) ([]domain.AppRecoveryInhibition, error) {
	572	+	inhibitions, err := s.deps.State.LoadRecoveryInhibitions(ctx, app)
	573	+	if err != nil {
	574	+	return nil, fmt.Errorf("deployment: load recovery inhibitions: %w", err)
	575	+	}
	576	+	_, pending := splitRecoveryInhibitions(app, inhibitions)
	577	+	return pending, nil
	578	+	}
	579	+
	580	+	// retirePendingForStop retires the pending superseded containers of a stopped
	581	+	// app, recording a leftover or a failure on the service they belong to. It
	582	+	// returns the services whose predecessor is not confirmed retired, and an
	583	+	// app-level error when the markers themselves could not be loaded: nothing
	584	+	// can then be said about any predecessor, which the caller must journal as a
	585	+	// failed operation.
	586	+	func (s Service) retirePendingForStop(ctx context.Context, app string, result LifecycleResult) ([]string, error) {
	587	+	pending, err := s.pendingRetirements(ctx, app)
	588	+	if err != nil {
	589	+	result.Warnings = append(result.Warnings, CleanupWarning{Detail: "pending retirements unknown: " + err.Error()})
	590	+	return []string{"pending retirements"}, err
	591	+	}
	592	+	var failed []string
	593	+	for _, marker := range pending {
	594	+	retired := s.retireContainer(ctx, app, retireOptions{
	595	+	Service: marker.Service, Grace: marker.StopGrace, ClearInhibition: true, RetryOnLeak: true,
	596	+	}, marker.ContainerID)
	597	+	svc := result.Services[marker.Service]
	598	+	svc.CleanupWarnings = append(svc.CleanupWarnings, retired.Warnings...)
	599	+	if !retired.Settled {
	600	+	svc.Result = "failed"
	601	+	svc.Error = "superseded container " + marker.ContainerID + " not retired: " + cleanupDetail(retired)
	602	+	failed = append(failed, marker.Service)
	603	+	}
	604	+	result.Services[marker.Service] = svc
	605	+	}
	606	+	return failed, nil
	607	+	}
	608	+
518	609		// ReconcileBoot reconciles apps intended to run after a daemon boot:
519	610		// every non-stopped app with ACTIVE services is verified through Start,
520	611		// including all-running apps: ephemeral loopback binds may shift across

@@ -691,6 +782,10 @@ func (s *Service) redeployPinned(ctx context.Context, app, opID, name string, ef
691	782		svcResult.Error = err.Error()
692	783		return svcResult, err
693	784		}
	785	+	// The execution is deliberately not recorded here: the graph is not
	786	+	// applied yet. Recording it before routing was accepted would let the next
	787	+	// periodic pass see a verified, unrouted generation and skip the
	788	+	// publication. Each caller records after its own refreshTraffic succeeded.
694	789		return svcResult, nil
695	790		}
696	791

@@ -709,6 +804,7 @@ func (s *Service) verifyRunningService(ctx context.Context, app, name string, ef
709	804		s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
710	805		return
711	806		}
	807	+	startedAt := s.executionStart(ctx, eff.Container)
712	808		binds, udpBinds, err := s.refreshBackendBinds(ctx, app, name, eff, s.deps.Traffic != nil)
713	809		if err != nil {
714	810		s.failServiceStep(ctx, app, name, eff, step, result, err.Error())

@@ -723,7 +819,7 @@ func (s *Service) verifyRunningService(ctx context.Context, app, name string, ef
723	819		// follows once, after the loop.
724	820		step.State = domain.AppStepSucceeded
725	821		step.After = eff.Container
726		-	result.Services[name] = ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds}
	822	+	result.Services[name] = ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds, VerifiedStartedAt: startedAt}
727	823		}
728	824
729	825		// refreshBackendBinds re-inspects the loopback publishes of a restarted


@@ -86,8 +86,22 @@ func (s Service) stopLocked(ctx context.Context, app, opID string) (LifecycleR
86		}	86		}
87		result.Services[name] = ServiceResult{Result: "deployed", Before: container, After: "", CleanupWarnings: warnings}	87		result.Services[name] = ServiceResult{Result: "deployed", Before: container, After: "", CleanupWarnings: warnings}
88		}	88		}
			89	+	// A superseded predecessor an overlap replacement left running is not in
			90	+	// ACTIVE, so the loop above never saw it: a stopped app must not keep it
			91	+	// running. One that cannot be retired keeps its marker and fails the stop.
			92	+	pendingFailed, pendingErr := s.retirePendingForStop(ctx, app, result)
			93	+	failures = append(failures, pendingFailed...)
89		op.Outcome = ComputeOutcome(result.Services)	94		op.Outcome = ComputeOutcome(result.Services)
90	-	op.Warnings = journalWarnings(collectCleanupWarnings(result.Services))	95	+	if pendingErr != nil {
			96	+	// An app-level cleanup failure belongs to no service, so the
			97	+	// per-service results alone would journal a success and a replay of
			98	+	// this key would read it. It is a failed step and a failed outcome.
			99	+	op.Steps = append(op.Steps, domain.AppOperationStep{
			100	+	ID: "pending.retire", State: domain.AppStepFailed, Error: pendingErr.Error(),
			101	+	})
			102	+	op.Outcome = domain.AppOutcomeFailed
			103	+	}
			104	+	op.Warnings = journalWarnings(append(collectCleanupWarnings(result.Services), result.Warnings...))
91		if err := s.deps.State.SaveOperation(ctx, op); err != nil {	105		if err := s.deps.State.SaveOperation(ctx, op); err != nil {
92		log.Warn().Err(err).Msg("deployment: failed to record stop outcome")	106		log.Warn().Err(err).Msg("deployment: failed to record stop outcome")
93		}	107		}

@@ -198,6 +212,10 @@ func (s Service) startLocked(ctx context.Context, app, opID string) (Lifecycle
198		s.failTrafficPublication(ctx, &op, err)	212		s.failTrafficPublication(ctx, &op, err)
199		return result, err	213		return result, err
200		}	214		}
			215	+	// Routing accepted the verified generations: record their executions.
			216	+	for name, svcResult := range result.Services {
			217	+	s.recordVerifiedExecution(app, name, svcResult)
			218	+	}
201		return result, nil	219		return result, nil
202		}	220		}
203			221

@@ -273,6 +291,11 @@ func (s *Service) restartLocked(ctx context.Context, app, service, opID string)
273		s.failTrafficPublication(ctx, &op, err)	291		s.failTrafficPublication(ctx, &op, err)
274		return result, err	292		return result, err
275		}	293		}
			294	+	// Routing accepted the restarted generations: their executions are
			295	+	// verified. The app lock is held, so no periodic pass ran in between.
			296	+	for name, svcResult := range result.Services {
			297	+	s.recordVerifiedExecution(app, name, svcResult)
			298	+	}
276		if len(failures) > 0 {	299		if len(failures) > 0 {
277		return result, fmt.Errorf("deployment: restart failed for %s: %w", strings.Join(failures, ", "), domain.ErrAppStateConflict)	300		return result, fmt.Errorf("deployment: restart failed for %s: %w", strings.Join(failures, ", "), domain.ErrAppStateConflict)
278		}	301		}

@@ -347,6 +370,8 @@ func (s *Service) restartOneService(ctx context.Context, app, opID, name string,
347		s.clearServiceBinds(ctx, app, name)	370		s.clearServiceBinds(ctx, app, name)
348		return fail(err.Error())	371		return fail(err.Error())
349		}	372		}
			373	+	// The restarted execution is observed before readiness runs.
			374	+	startedAt := s.executionStart(ctx, eff.Container)
350		// Ephemeral loopback binds are not guaranteed stable across a runtime	375		// Ephemeral loopback binds are not guaranteed stable across a runtime
351		// restart: re-inspect before probing so the proxy never dials a stale	376		// restart: re-inspect before probing so the proxy never dials a stale
352		// bind.	377		// bind.

@@ -361,7 +386,7 @@ func (s *Service) restartOneService(ctx context.Context, app, opID, name string,
361		}	386		}
362		step.State = domain.AppStepSucceeded	387		step.State = domain.AppStepSucceeded
363		step.After = eff.Container	388		step.After = eff.Container
364	-	return ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds}	389	+	return ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds, VerifiedStartedAt: startedAt}
365		}	390		}
366			391
367		// Remove withdraws workloads by exact container ID; volumes, secrets, and	392		// Remove withdraws workloads by exact container ID; volumes, secrets, and

@@ -512,9 +537,75 @@ func (s *Service) removeServiceContainers(ctx context.Context, app, opID string,
512		step.State = domain.AppStepSucceeded	537		step.State = domain.AppStepSucceeded
513		steps = append(steps, step)	538		steps = append(steps, step)
514		}	539		}
			540	+	if err := s.retireAllPending(ctx, app); err != nil {
			541	+	return nil, err
			542	+	}
515		return steps, nil	543		return steps, nil
516		}	544		}
517			545
			546	+	// retireAllPending retires every superseded container an overlap replacement
			547	+	// left running beside its successor. Ending the incarnation deletes the
			548	+	// retirement markers with the rest of the app state, so a container that is
			549	+	// not confirmed gone here would be orphaned and untracked: a failure keeps the
			550	+	// markers and fails the caller. Every marker is attempted, and a marker whose
			551	+	// claims could not be released is a failure too, so it is retried instead of
			552	+	// leaking its claims.
			553	+	func (s *Service) retireAllPending(ctx context.Context, app string) error {
			554	+	pending, err := s.pendingRetirements(ctx, app)
			555	+	if err != nil {
			556	+	return err
			557	+	}
			558	+	var failures []error
			559	+	for _, marker := range pending {
			560	+	retired := s.retireContainer(ctx, app, retireOptions{
			561	+	Service: marker.Service, Grace: marker.StopGrace, ClearInhibition: true, RetryOnLeak: true,
			562	+	}, marker.ContainerID)
			563	+	if !retired.Settled {
			564	+	failures = append(failures, fmt.Errorf("deployment: remove %q/%q: superseded container %s not retired: %s", app, marker.Service, marker.ContainerID, cleanupDetail(retired)))
			565	+	}
			566	+	}
			567	+	return errors.Join(failures...)
			568	+	}
			569	+
			570	+	// pendingRetirements loads the retirement-pending markers of one app.
			571	+	func (s *Service) pendingRetirements(ctx context.Context, app string) ([]domain.AppRecoveryInhibition, error) {
			572	+	inhibitions, err := s.deps.State.LoadRecoveryInhibitions(ctx, app)
			573	+	if err != nil {
			574	+	return nil, fmt.Errorf("deployment: load recovery inhibitions: %w", err)
			575	+	}
			576	+	_, pending := splitRecoveryInhibitions(app, inhibitions)
			577	+	return pending, nil
			578	+	}
			579	+
			580	+	// retirePendingForStop retires the pending superseded containers of a stopped
			581	+	// app, recording a leftover or a failure on the service they belong to. It
			582	+	// returns the services whose predecessor is not confirmed retired, and an
			583	+	// app-level error when the markers themselves could not be loaded: nothing
			584	+	// can then be said about any predecessor, which the caller must journal as a
			585	+	// failed operation.
			586	+	func (s Service) retirePendingForStop(ctx context.Context, app string, result LifecycleResult) ([]string, error) {
			587	+	pending, err := s.pendingRetirements(ctx, app)
			588	+	if err != nil {
			589	+	result.Warnings = append(result.Warnings, CleanupWarning{Detail: "pending retirements unknown: " + err.Error()})
			590	+	return []string{"pending retirements"}, err
			591	+	}
			592	+	var failed []string
			593	+	for _, marker := range pending {
			594	+	retired := s.retireContainer(ctx, app, retireOptions{
			595	+	Service: marker.Service, Grace: marker.StopGrace, ClearInhibition: true, RetryOnLeak: true,
			596	+	}, marker.ContainerID)
			597	+	svc := result.Services[marker.Service]
			598	+	svc.CleanupWarnings = append(svc.CleanupWarnings, retired.Warnings...)
			599	+	if !retired.Settled {
			600	+	svc.Result = "failed"
			601	+	svc.Error = "superseded container " + marker.ContainerID + " not retired: " + cleanupDetail(retired)
			602	+	failed = append(failed, marker.Service)
			603	+	}
			604	+	result.Services[marker.Service] = svc
			605	+	}
			606	+	return failed, nil
			607	+	}
			608	+
518		// ReconcileBoot reconciles apps intended to run after a daemon boot:	609		// ReconcileBoot reconciles apps intended to run after a daemon boot:
519		// every non-stopped app with ACTIVE services is verified through Start,	610		// every non-stopped app with ACTIVE services is verified through Start,
520		// including all-running apps: ephemeral loopback binds may shift across	611		// including all-running apps: ephemeral loopback binds may shift across

@@ -691,6 +782,10 @@ func (s *Service) redeployPinned(ctx context.Context, app, opID, name string, ef
691		svcResult.Error = err.Error()	782		svcResult.Error = err.Error()
692		return svcResult, err	783		return svcResult, err
693		}	784		}
			785	+	// The execution is deliberately not recorded here: the graph is not
			786	+	// applied yet. Recording it before routing was accepted would let the next
			787	+	// periodic pass see a verified, unrouted generation and skip the
			788	+	// publication. Each caller records after its own refreshTraffic succeeded.
694		return svcResult, nil	789		return svcResult, nil
695		}	790		}
696			791

@@ -709,6 +804,7 @@ func (s *Service) verifyRunningService(ctx context.Context, app, name string, ef
709		s.failServiceStep(ctx, app, name, eff, step, result, err.Error())	804		s.failServiceStep(ctx, app, name, eff, step, result, err.Error())
710		return	805		return
711		}	806		}
			807	+	startedAt := s.executionStart(ctx, eff.Container)
712		binds, udpBinds, err := s.refreshBackendBinds(ctx, app, name, eff, s.deps.Traffic != nil)	808		binds, udpBinds, err := s.refreshBackendBinds(ctx, app, name, eff, s.deps.Traffic != nil)
713		if err != nil {	809		if err != nil {
714		s.failServiceStep(ctx, app, name, eff, step, result, err.Error())	810		s.failServiceStep(ctx, app, name, eff, step, result, err.Error())

@@ -723,7 +819,7 @@ func (s *Service) verifyRunningService(ctx context.Context, app, name string, ef
723		// follows once, after the loop.	819		// follows once, after the loop.
724		step.State = domain.AppStepSucceeded	820		step.State = domain.AppStepSucceeded
725		step.After = eff.Container	821		step.After = eff.Container
726	-	result.Services[name] = ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds}	822	+	result.Services[name] = ServiceResult{Result: "deployed", Before: eff.Container, After: eff.Container, BackendBinds: binds, UDPBackendBinds: udpBinds, VerifiedStartedAt: startedAt}
727		}	823		}
728			824
729		// refreshBackendBinds re-inspects the loopback publishes of a restarted	825		// refreshBackendBinds re-inspects the loopback publishes of a restarted

M internal/usecase/deployment/monitor.go +146 -23


@@ -295,6 +295,32 @@ func (t *executionTracker) forget(key recoveryKey) {
295	295		delete(t.seen, key)
296	296		}
297	297
	298	+	// executionStart returns the start time of the container's current
	299	+	// execution, or the zero time when it cannot be read: an unknown execution is
	300	+	// simply never recorded as verified, so the periodic pass verifies it.
	301	+	func (s *Service) executionStart(ctx context.Context, containerID string) time.Time {
	302	+	if s.deps.Runtime == nil \|\| containerID == "" {
	303	+	return time.Time{}
	304	+	}
	305	+	container, err := s.deps.Runtime.InspectContainer(ctx, containerID)
	306	+	if err != nil \|\| container == nil {
	307	+	return time.Time{}
	308	+	}
	309	+	return container.StartedAt
	310	+	}
	311	+
	312	+	// recordVerifiedExecution records the execution a deploy or restart verified
	313	+	// and routed, so the first periodic pass after it does not treat the new
	314	+	// generation as an unseen execution, withdraw it, and verify it again. Only a
	315	+	// successful, non-zero observation is recorded; failure paths never are, and
	316	+	// a later native restart still differs from the recorded start time.
	317	+	func (s *Service) recordVerifiedExecution(app, service string, result ServiceResult) {
	318	+	if result.Result == "failed" \|\| result.After == "" \|\| result.VerifiedStartedAt.IsZero() {
	319	+	return
	320	+	}
	321	+	s.executions.record(recoveryKey{app: app, service: service, container: result.After}, result.VerifiedStartedAt)
	322	+	}
	323	+
298	324		// ReconcileRunning is the daemon-owned periodic recovery pass. It never
299	325		// consults DESIRED to select content, never changes intent, never
300	326		// pulls, creates, or removes a container, and never starts a

@@ -358,7 +384,11 @@ func (s *Service) reconcileAppRunning(ctx context.Context, app string) error {
358	384		if err != nil {
359	385		return fmt.Errorf("deployment: reconcile %q: load recovery inhibitions: %w", app, err)
360	386		}
361		-	inhibited, failures := s.processRecoveryInhibitions(ctx, app, inhibitions)
	387	+	inhibited, pending := splitRecoveryInhibitions(app, inhibitions)
	388	+	var failures []error
	389	+	// verified records, per service, the execution this pass verified and
	390	+	// routed. A successor that is not in it is not known to serve.
	391	+	verified := map[string]time.Time{}
362	392		for _, name := range sortedServiceNames(active) {
363	393		eff := active.Services[name]
364	394		if intent.Stopped {

@@ -367,9 +397,13 @@ func (s *Service) reconcileAppRunning(ctx context.Context, app string) error {
367	397		}
368	398		continue
369	399		}
370		-	if err := s.reconcileRunningService(ctx, app, name, eff, inhibited); err != nil {
	400	+	startedAt, ok, err := s.reconcileRunningService(ctx, app, name, eff, inhibited)
	401	+	if err != nil {
371	402		failures = append(failures, err)
372	403		}
	404	+	if ok {
	405	+	verified[name] = startedAt
	406	+	}
373	407		}
374	408		if intent.Stopped {
375	409		// A stopped app must not stay published: rebuild the projection

@@ -381,24 +415,86 @@ func (s *Service) reconcileAppRunning(ctx context.Context, app string) error {
381	415		}
382	416		cancel()
383	417		}
	418	+	// Superseded containers are retired last: only after their successors
	419	+	// were verified and routed in this pass.
	420	+	failures = append(failures, s.retirePendingContainers(ctx, app, intent.Stopped, active, pending, verified)...)
384	421		s.pruneRecoveryHistory(app, active)
385	422		return errors.Join(failures...)
386	423		}
387	424
388		-	func (s *Service) processRecoveryInhibitions(ctx context.Context, app string, inhibitions []domain.AppRecoveryInhibition) (map[recoveryKey]struct{}, []error) {
	425	+	// splitRecoveryInhibitions separates the generations recovery must never
	426	+	// start from the superseded containers waiting to be retired.
	427	+	func splitRecoveryInhibitions(app string, inhibitions []domain.AppRecoveryInhibition) (map[recoveryKey]struct{}, []domain.AppRecoveryInhibition) {
389	428		inhibited := map[recoveryKey]struct{}{}
390		-	var failures []error
	429	+	var pending []domain.AppRecoveryInhibition
391	430		for _, inhibition := range inhibitions {
392		-	if inhibition.Reason != domain.AppInhibitRetirementPending {
393		-	inhibited[recoveryKey{app: app, service: inhibition.Service, container: inhibition.ContainerID}] = struct{}{}
	431	+	if inhibition.Reason == domain.AppInhibitRetirementPending {
	432	+	pending = append(pending, inhibition)
	433	+	continue
	434	+	}
	435	+	inhibited[recoveryKey{app: app, service: inhibition.Service, container: inhibition.ContainerID}] = struct{}{}
	436	+	}
	437	+	return inhibited, pending
	438	+	}
	439	+
	440	+	// retirePendingContainers retires the superseded containers an overlap
	441	+	// replacement left running, with the stop grace recorded at the marker
	442	+	// (the app default for a record that has none). A container is retired only
	443	+	// when it can no longer be the serving one:
	444	+	// - ACTIVE names it (the replacement never took over): it is the serving
	445	+	// generation, so the marker is dropped and the container is kept;
	446	+	// - its successor was not verified and routed by THIS pass, or restarted
	447	+	// since: the superseded container is retried later, so a successor that
	448	+	// is not serving never costs the working generation.
	449	+	//
	450	+	// A stopped app has no successor to wait for: nothing may run. Cleanup
	451	+	// warnings of a retirement (claims not released, secret files) are logged and
	452	+	// returned, and the marker stays until the container is gone with its claims
	453	+	// released, so the next pass retries.
	454	+	func (s *Service) retirePendingContainers(ctx context.Context, app string, stopped bool, active domain.AppActive, pending []domain.AppRecoveryInhibition, verified map[string]time.Time) []error {
	455	+	var failures []error
	456	+	for _, marker := range pending {
	457	+	if publishedContainer(active, marker.ContainerID) {
	458	+	if err := s.clearRecoveryInhibition(ctx, app, marker.Service, marker.ContainerID); err != nil {
	459	+	failures = append(failures, err)
	460	+	}
	461	+	continue
	462	+	}
	463	+	if !stopped && !s.successorVerified(ctx, app, marker.Service, active, verified) {
	464	+	continue
	465	+	}
	466	+	retired := s.retireContainer(ctx, app, retireOptions{
	467	+	Service: marker.Service, Grace: marker.StopGrace, ClearInhibition: true, RetryOnLeak: true,
	468	+	}, marker.ContainerID)
	469	+	if !retired.Settled {
	470	+	failures = append(failures, fmt.Errorf("deployment: retirement pending for %s/%s: %s", app, marker.ContainerID, cleanupDetail(retired)))
394	471		continue
395	472		}
396		-	retired := s.retireContainer(ctx, app, retireOptions{Service: inhibition.Service}, inhibition.ContainerID)
397		-	if !retired.Gone {
398		-	failures = append(failures, fmt.Errorf("deployment: retirement pending for %s/%s", app, inhibition.ContainerID))
	473	+	for _, warning := range retired.Warnings {
	474	+	s.log.Warn().Str("app", app).Str("service", warning.Service).Str("container", warning.Leftover).
	475	+	Str("detail", warning.Detail).Msg("deployment: superseded container retired with leftovers")
	476	+	failures = append(failures, fmt.Errorf("deployment: retired superseded container %s/%s with leftovers: %s", app, warning.Leftover, warning.Detail))
399	477		}
400	478		}
401		-	return inhibited, failures
	479	+	return failures
	480	+	}
	481	+
	482	+	// successorVerified reports whether the container ACTIVE names for the
	483	+	// service was verified and routed by this very pass, and is still the same
	484	+	// running execution: a final inspection that shows a different start time
	485	+	// means the successor restarted after it was verified, so it is unverified
	486	+	// again. A service ACTIVE no longer carries has no successor to wait for.
	487	+	func (s *Service) successorVerified(ctx context.Context, app, service string, active domain.AppActive, verified map[string]time.Time) bool {
	488	+	successor, ok := active.Services[service]
	489	+	if !ok \|\| successor.Container == "" {
	490	+	return true
	491	+	}
	492	+	startedAt, ok := verified[service]
	493	+	if !ok \|\| s.deps.Runtime == nil {
	494	+	return false
	495	+	}
	496	+	container, err := s.deps.Runtime.InspectContainer(ctx, successor.Container)
	497	+	return err == nil && container.Status == string(domain.ContainerStatusRunning) && container.StartedAt.Equal(startedAt)
402	498		}
403	499
404	500		// convergeStoppedService enforces durable stopped intent: it never

@@ -433,23 +529,23 @@ func (s *Service) convergeStoppedService(ctx context.Context, app, name string,
433	529
434	530		// reconcileRunningService recovers one service to the running intent
435	531		// carried by ACTIVE. It only ever acts on the exact ACTIVE container ID.
436		-	func (s *Service) reconcileRunningService(ctx context.Context, app, name string, eff domain.AppEffectiveService, inhibited map[recoveryKey]struct{}) error {
	532	+	func (s *Service) reconcileRunningService(ctx context.Context, app, name string, eff domain.AppEffectiveService, inhibited map[recoveryKey]struct{}) (time.Time, bool, error) {
437	533		if eff.Container == "" {
438	534		// Absent ACTIVE: never act, never reconstruct.
439		-	return nil
	535	+	return time.Time{}, false, nil
440	536		}
441	537		if _, blocked := inhibited[recoveryKey{app: app, service: name, container: eff.Container}]; blocked {
442		-	return s.refuseInhibitedRecovery(ctx, app, name, eff.Container)
	538	+	return time.Time{}, false, s.refuseInhibitedRecovery(ctx, app, name, eff.Container)
443	539		}
444	540		if s.deps.Runtime == nil {
445		-	return nil
	541	+	return time.Time{}, false, nil
446	542		}
447	543		serviceCtx, cancel := context.WithTimeout(ctx, reconcileServiceDeadline)
448	544		defer cancel()
449	545		key := recoveryKey{app: app, service: name, container: eff.Container}
450	546		container, fresh, err := s.inspectForRecovery(serviceCtx, app, name, eff, key)
451	547		if err != nil {
452		-	return err
	548	+	return time.Time{}, false, err
453	549		}
454	550		return s.convergeRecoveredService(serviceCtx, app, name, eff, container, key, fresh)
455	551		}

@@ -494,30 +590,31 @@ func (s *Service) inspectForRecovery(ctx context.Context, app, name string, eff
494	590		// convergeRecoveredService drives the exact ACTIVE container to a
495	591		// verified, published state, or leaves it withdrawn with a reported
496	592		// reason.
497		-	func (s Service) convergeRecoveredService(ctx context.Context, app, name string, eff domain.AppEffectiveService, container domain.Container, key recoveryKey, fresh bool) error {
	593	+	func (s Service) convergeRecoveredService(ctx context.Context, app, name string, eff domain.AppEffectiveService, container domain.Container, key recoveryKey, fresh bool) (time.Time, bool, error) {
498	594		started, verify, err := s.driveRecoveredContainer(ctx, app, name, eff, container, key)
499	595		if err != nil {
500		-	return errors.Join(s.withdrawForRecovery(ctx, app, name), err)
	596	+	return time.Time{}, false, errors.Join(s.withdrawForRecovery(ctx, app, name), err)
501	597		}
502	598		if !verify {
503	599		// Transitional or backoff-delayed: observe now, converge later.
504	600		s.backoff.invalidateStable(key)
505		-	return nil
	601	+	return time.Time{}, false, nil
506	602		}
507	603		restarted, stable, err := s.recoverHealth(ctx, app, name, eff, key)
508	604		if err != nil {
509	605		s.backoff.invalidateStable(key)
510		-	return errors.Join(s.withdrawForRecovery(ctx, app, name), err)
	606	+	return time.Time{}, false, errors.Join(s.withdrawForRecovery(ctx, app, name), err)
511	607		}
512	608		if err := s.publishRecoveredService(ctx, app, name, eff, container, key, started, restarted, fresh); err != nil {
513		-	return err
	609	+	return time.Time{}, false, err
514	610		}
515	611		if stable {
516	612		s.backoff.observeStable(key)
517	613		} else {
518	614		s.backoff.invalidateStable(key)
519	615		}
520		-	return nil
	616	+	// Verified and published: report the execution this pass vouches for.
	617	+	return container.StartedAt, true, nil
521	618		}
522	619
523	620		// refuseInhibitedRecovery reports a generation whose recovery is durably

@@ -762,7 +859,10 @@ func (s *Service) withdrawForRecovery(ctx context.Context, app, service string)
762	859		}
763	860
764	861		// recoveryInhibited reports whether one exact generation is durably
765		-	// inhibited for recovery.
	862	+	// inhibited for recovery. A retirement-pending marker does not count: it
	863	+	// names a superseded container that is being retired, not a generation
	864	+	// recovery must keep stopped, and when ACTIVE still names that container
	865	+	// (the replacement never took over) it must stay startable.
766	866		func (s *Service) recoveryInhibited(ctx context.Context, app, service, containerID string) (bool, error) {
767	867		if containerID == "" {
768	868		return false, nil

@@ -772,7 +872,8 @@ func (s *Service) recoveryInhibited(ctx context.Context, app, service, container
772	872		return false, fmt.Errorf("deployment: load recovery inhibitions: %w", err)
773	873		}
774	874		for _, inhibition := range inhibitions {
775		-	if inhibition.Service == service && inhibition.ContainerID == containerID {
	875	+	if inhibition.Service == service && inhibition.ContainerID == containerID &&
	876	+	inhibition.Reason != domain.AppInhibitRetirementPending {
776	877		return true, nil
777	878		}
778	879		}

@@ -799,6 +900,28 @@ func (s *Service) inhibitRecovery(ctx context.Context, app, service, containerID
799	900		return nil
800	901		}
801	902
	903	+	// inhibitRetirement durably marks a superseded container that an overlap
	904	+	// replacement left running beside its successor. The periodic reconciliation
	905	+	// retires it with grace, once the successor is verified and routed.
	906	+	func (s *Service) inhibitRetirement(ctx context.Context, app, service, containerID, operation string, grace time.Duration) error {
	907	+	if containerID == "" {
	908	+	return nil
	909	+	}
	910	+	inhibition := domain.AppRecoveryInhibition{
	911	+	App: app,
	912	+	Service: service,
	913	+	ContainerID: containerID,
	914	+	Reason: domain.AppInhibitRetirementPending,
	915	+	Operation: operation,
	916	+	CreatedAt: time.Now().UTC(),
	917	+	StopGrace: grace,
	918	+	}
	919	+	if err := s.deps.State.SaveRecoveryInhibition(ctx, inhibition); err != nil {
	920	+	return fmt.Errorf("deployment: record retirement of %s/%s: %w", app, containerID, err)
	921	+	}
	922	+	return nil
	923	+	}
	924	+
802	925		// clearRecoveryInhibition drops the inhibition of a safely superseded
803	926		// generation.
804	927		func (s *Service) clearRecoveryInhibition(ctx context.Context, app, service, containerID string) error {


@@ -295,6 +295,32 @@ func (t *executionTracker) forget(key recoveryKey) {
295		delete(t.seen, key)	295		delete(t.seen, key)
296		}	296		}
297			297
			298	+	// executionStart returns the start time of the container's current
			299	+	// execution, or the zero time when it cannot be read: an unknown execution is
			300	+	// simply never recorded as verified, so the periodic pass verifies it.
			301	+	func (s *Service) executionStart(ctx context.Context, containerID string) time.Time {
			302	+	if s.deps.Runtime == nil \|\| containerID == "" {
			303	+	return time.Time{}
			304	+	}
			305	+	container, err := s.deps.Runtime.InspectContainer(ctx, containerID)
			306	+	if err != nil \|\| container == nil {
			307	+	return time.Time{}
			308	+	}
			309	+	return container.StartedAt
			310	+	}
			311	+
			312	+	// recordVerifiedExecution records the execution a deploy or restart verified
			313	+	// and routed, so the first periodic pass after it does not treat the new
			314	+	// generation as an unseen execution, withdraw it, and verify it again. Only a
			315	+	// successful, non-zero observation is recorded; failure paths never are, and
			316	+	// a later native restart still differs from the recorded start time.
			317	+	func (s *Service) recordVerifiedExecution(app, service string, result ServiceResult) {
			318	+	if result.Result == "failed" \|\| result.After == "" \|\| result.VerifiedStartedAt.IsZero() {
			319	+	return
			320	+	}
			321	+	s.executions.record(recoveryKey{app: app, service: service, container: result.After}, result.VerifiedStartedAt)
			322	+	}
			323	+
298		// ReconcileRunning is the daemon-owned periodic recovery pass. It never	324		// ReconcileRunning is the daemon-owned periodic recovery pass. It never
299		// consults DESIRED to select content, never changes intent, never	325		// consults DESIRED to select content, never changes intent, never
300		// pulls, creates, or removes a container, and never starts a	326		// pulls, creates, or removes a container, and never starts a

@@ -358,7 +384,11 @@ func (s *Service) reconcileAppRunning(ctx context.Context, app string) error {
358		if err != nil {	384		if err != nil {
359		return fmt.Errorf("deployment: reconcile %q: load recovery inhibitions: %w", app, err)	385		return fmt.Errorf("deployment: reconcile %q: load recovery inhibitions: %w", app, err)
360		}	386		}
361	-	inhibited, failures := s.processRecoveryInhibitions(ctx, app, inhibitions)	387	+	inhibited, pending := splitRecoveryInhibitions(app, inhibitions)
			388	+	var failures []error
			389	+	// verified records, per service, the execution this pass verified and
			390	+	// routed. A successor that is not in it is not known to serve.
			391	+	verified := map[string]time.Time{}
362		for _, name := range sortedServiceNames(active) {	392		for _, name := range sortedServiceNames(active) {
363		eff := active.Services[name]	393		eff := active.Services[name]
364		if intent.Stopped {	394		if intent.Stopped {

@@ -367,9 +397,13 @@ func (s *Service) reconcileAppRunning(ctx context.Context, app string) error {
367		}	397		}
368		continue	398		continue
369		}	399		}
370	-	if err := s.reconcileRunningService(ctx, app, name, eff, inhibited); err != nil {	400	+	startedAt, ok, err := s.reconcileRunningService(ctx, app, name, eff, inhibited)
			401	+	if err != nil {
371		failures = append(failures, err)	402		failures = append(failures, err)
372		}	403		}
			404	+	if ok {
			405	+	verified[name] = startedAt
			406	+	}
373		}	407		}
374		if intent.Stopped {	408		if intent.Stopped {
375		// A stopped app must not stay published: rebuild the projection	409		// A stopped app must not stay published: rebuild the projection

@@ -381,24 +415,86 @@ func (s *Service) reconcileAppRunning(ctx context.Context, app string) error {
381		}	415		}
382		cancel()	416		cancel()
383		}	417		}
			418	+	// Superseded containers are retired last: only after their successors
			419	+	// were verified and routed in this pass.
			420	+	failures = append(failures, s.retirePendingContainers(ctx, app, intent.Stopped, active, pending, verified)...)
384		s.pruneRecoveryHistory(app, active)	421		s.pruneRecoveryHistory(app, active)
385		return errors.Join(failures...)	422		return errors.Join(failures...)
386		}	423		}
387			424
388	-	func (s *Service) processRecoveryInhibitions(ctx context.Context, app string, inhibitions []domain.AppRecoveryInhibition) (map[recoveryKey]struct{}, []error) {	425	+	// splitRecoveryInhibitions separates the generations recovery must never
			426	+	// start from the superseded containers waiting to be retired.
			427	+	func splitRecoveryInhibitions(app string, inhibitions []domain.AppRecoveryInhibition) (map[recoveryKey]struct{}, []domain.AppRecoveryInhibition) {
389		inhibited := map[recoveryKey]struct{}{}	428		inhibited := map[recoveryKey]struct{}{}
390	-	var failures []error	429	+	var pending []domain.AppRecoveryInhibition
391		for _, inhibition := range inhibitions {	430		for _, inhibition := range inhibitions {
392	-	if inhibition.Reason != domain.AppInhibitRetirementPending {	431	+	if inhibition.Reason == domain.AppInhibitRetirementPending {
393	-	inhibited[recoveryKey{app: app, service: inhibition.Service, container: inhibition.ContainerID}] = struct{}{}	432	+	pending = append(pending, inhibition)
			433	+	continue
			434	+	}
			435	+	inhibited[recoveryKey{app: app, service: inhibition.Service, container: inhibition.ContainerID}] = struct{}{}
			436	+	}
			437	+	return inhibited, pending
			438	+	}
			439	+
			440	+	// retirePendingContainers retires the superseded containers an overlap
			441	+	// replacement left running, with the stop grace recorded at the marker
			442	+	// (the app default for a record that has none). A container is retired only
			443	+	// when it can no longer be the serving one:
			444	+	// - ACTIVE names it (the replacement never took over): it is the serving
			445	+	// generation, so the marker is dropped and the container is kept;
			446	+	// - its successor was not verified and routed by THIS pass, or restarted
			447	+	// since: the superseded container is retried later, so a successor that
			448	+	// is not serving never costs the working generation.
			449	+	//
			450	+	// A stopped app has no successor to wait for: nothing may run. Cleanup
			451	+	// warnings of a retirement (claims not released, secret files) are logged and
			452	+	// returned, and the marker stays until the container is gone with its claims
			453	+	// released, so the next pass retries.
			454	+	func (s *Service) retirePendingContainers(ctx context.Context, app string, stopped bool, active domain.AppActive, pending []domain.AppRecoveryInhibition, verified map[string]time.Time) []error {
			455	+	var failures []error
			456	+	for _, marker := range pending {
			457	+	if publishedContainer(active, marker.ContainerID) {
			458	+	if err := s.clearRecoveryInhibition(ctx, app, marker.Service, marker.ContainerID); err != nil {
			459	+	failures = append(failures, err)
			460	+	}
			461	+	continue
			462	+	}
			463	+	if !stopped && !s.successorVerified(ctx, app, marker.Service, active, verified) {
			464	+	continue
			465	+	}
			466	+	retired := s.retireContainer(ctx, app, retireOptions{
			467	+	Service: marker.Service, Grace: marker.StopGrace, ClearInhibition: true, RetryOnLeak: true,
			468	+	}, marker.ContainerID)
			469	+	if !retired.Settled {
			470	+	failures = append(failures, fmt.Errorf("deployment: retirement pending for %s/%s: %s", app, marker.ContainerID, cleanupDetail(retired)))
394		continue	471		continue
395		}	472		}
396	-	retired := s.retireContainer(ctx, app, retireOptions{Service: inhibition.Service}, inhibition.ContainerID)	473	+	for _, warning := range retired.Warnings {
397	-	if !retired.Gone {	474	+	s.log.Warn().Str("app", app).Str("service", warning.Service).Str("container", warning.Leftover).
398	-	failures = append(failures, fmt.Errorf("deployment: retirement pending for %s/%s", app, inhibition.ContainerID))	475	+	Str("detail", warning.Detail).Msg("deployment: superseded container retired with leftovers")
			476	+	failures = append(failures, fmt.Errorf("deployment: retired superseded container %s/%s with leftovers: %s", app, warning.Leftover, warning.Detail))
399		}	477		}
400		}	478		}
401	-	return inhibited, failures	479	+	return failures
			480	+	}
			481	+
			482	+	// successorVerified reports whether the container ACTIVE names for the
			483	+	// service was verified and routed by this very pass, and is still the same
			484	+	// running execution: a final inspection that shows a different start time
			485	+	// means the successor restarted after it was verified, so it is unverified
			486	+	// again. A service ACTIVE no longer carries has no successor to wait for.
			487	+	func (s *Service) successorVerified(ctx context.Context, app, service string, active domain.AppActive, verified map[string]time.Time) bool {
			488	+	successor, ok := active.Services[service]
			489	+	if !ok \|\| successor.Container == "" {
			490	+	return true
			491	+	}
			492	+	startedAt, ok := verified[service]
			493	+	if !ok \|\| s.deps.Runtime == nil {
			494	+	return false
			495	+	}
			496	+	container, err := s.deps.Runtime.InspectContainer(ctx, successor.Container)
			497	+	return err == nil && container.Status == string(domain.ContainerStatusRunning) && container.StartedAt.Equal(startedAt)
402		}	498		}
403			499
404		// convergeStoppedService enforces durable stopped intent: it never	500		// convergeStoppedService enforces durable stopped intent: it never

@@ -433,23 +529,23 @@ func (s *Service) convergeStoppedService(ctx context.Context, app, name string,
433			529
434		// reconcileRunningService recovers one service to the running intent	530		// reconcileRunningService recovers one service to the running intent
435		// carried by ACTIVE. It only ever acts on the exact ACTIVE container ID.	531		// carried by ACTIVE. It only ever acts on the exact ACTIVE container ID.
436	-	func (s *Service) reconcileRunningService(ctx context.Context, app, name string, eff domain.AppEffectiveService, inhibited map[recoveryKey]struct{}) error {	532	+	func (s *Service) reconcileRunningService(ctx context.Context, app, name string, eff domain.AppEffectiveService, inhibited map[recoveryKey]struct{}) (time.Time, bool, error) {
437		if eff.Container == "" {	533		if eff.Container == "" {
438		// Absent ACTIVE: never act, never reconstruct.	534		// Absent ACTIVE: never act, never reconstruct.
439	-	return nil	535	+	return time.Time{}, false, nil
440		}	536		}
441		if _, blocked := inhibited[recoveryKey{app: app, service: name, container: eff.Container}]; blocked {	537		if _, blocked := inhibited[recoveryKey{app: app, service: name, container: eff.Container}]; blocked {
442	-	return s.refuseInhibitedRecovery(ctx, app, name, eff.Container)	538	+	return time.Time{}, false, s.refuseInhibitedRecovery(ctx, app, name, eff.Container)
443		}	539		}
444		if s.deps.Runtime == nil {	540		if s.deps.Runtime == nil {
445	-	return nil	541	+	return time.Time{}, false, nil
446		}	542		}
447		serviceCtx, cancel := context.WithTimeout(ctx, reconcileServiceDeadline)	543		serviceCtx, cancel := context.WithTimeout(ctx, reconcileServiceDeadline)
448		defer cancel()	544		defer cancel()
449		key := recoveryKey{app: app, service: name, container: eff.Container}	545		key := recoveryKey{app: app, service: name, container: eff.Container}
450		container, fresh, err := s.inspectForRecovery(serviceCtx, app, name, eff, key)	546		container, fresh, err := s.inspectForRecovery(serviceCtx, app, name, eff, key)
451		if err != nil {	547		if err != nil {
452	-	return err	548	+	return time.Time{}, false, err
453		}	549		}
454		return s.convergeRecoveredService(serviceCtx, app, name, eff, container, key, fresh)	550		return s.convergeRecoveredService(serviceCtx, app, name, eff, container, key, fresh)
455		}	551		}

@@ -494,30 +590,31 @@ func (s *Service) inspectForRecovery(ctx context.Context, app, name string, eff
494		// convergeRecoveredService drives the exact ACTIVE container to a	590		// convergeRecoveredService drives the exact ACTIVE container to a
495		// verified, published state, or leaves it withdrawn with a reported	591		// verified, published state, or leaves it withdrawn with a reported
496		// reason.	592		// reason.
497	-	func (s Service) convergeRecoveredService(ctx context.Context, app, name string, eff domain.AppEffectiveService, container domain.Container, key recoveryKey, fresh bool) error {	593	+	func (s Service) convergeRecoveredService(ctx context.Context, app, name string, eff domain.AppEffectiveService, container domain.Container, key recoveryKey, fresh bool) (time.Time, bool, error) {
498		started, verify, err := s.driveRecoveredContainer(ctx, app, name, eff, container, key)	594		started, verify, err := s.driveRecoveredContainer(ctx, app, name, eff, container, key)
499		if err != nil {	595		if err != nil {
500	-	return errors.Join(s.withdrawForRecovery(ctx, app, name), err)	596	+	return time.Time{}, false, errors.Join(s.withdrawForRecovery(ctx, app, name), err)
501		}	597		}
502		if !verify {	598		if !verify {
503		// Transitional or backoff-delayed: observe now, converge later.	599		// Transitional or backoff-delayed: observe now, converge later.
504		s.backoff.invalidateStable(key)	600		s.backoff.invalidateStable(key)
505	-	return nil	601	+	return time.Time{}, false, nil
506		}	602		}
507		restarted, stable, err := s.recoverHealth(ctx, app, name, eff, key)	603		restarted, stable, err := s.recoverHealth(ctx, app, name, eff, key)
508		if err != nil {	604		if err != nil {
509		s.backoff.invalidateStable(key)	605		s.backoff.invalidateStable(key)
510	-	return errors.Join(s.withdrawForRecovery(ctx, app, name), err)	606	+	return time.Time{}, false, errors.Join(s.withdrawForRecovery(ctx, app, name), err)
511		}	607		}
512		if err := s.publishRecoveredService(ctx, app, name, eff, container, key, started, restarted, fresh); err != nil {	608		if err := s.publishRecoveredService(ctx, app, name, eff, container, key, started, restarted, fresh); err != nil {
513	-	return err	609	+	return time.Time{}, false, err
514		}	610		}
515		if stable {	611		if stable {
516		s.backoff.observeStable(key)	612		s.backoff.observeStable(key)
517		} else {	613		} else {
518		s.backoff.invalidateStable(key)	614		s.backoff.invalidateStable(key)
519		}	615		}
520	-	return nil	616	+	// Verified and published: report the execution this pass vouches for.
			617	+	return container.StartedAt, true, nil
521		}	618		}
522			619
523		// refuseInhibitedRecovery reports a generation whose recovery is durably	620		// refuseInhibitedRecovery reports a generation whose recovery is durably

@@ -762,7 +859,10 @@ func (s *Service) withdrawForRecovery(ctx context.Context, app, service string)
762		}	859		}
763			860
764		// recoveryInhibited reports whether one exact generation is durably	861		// recoveryInhibited reports whether one exact generation is durably
765	-	// inhibited for recovery.	862	+	// inhibited for recovery. A retirement-pending marker does not count: it
			863	+	// names a superseded container that is being retired, not a generation
			864	+	// recovery must keep stopped, and when ACTIVE still names that container
			865	+	// (the replacement never took over) it must stay startable.
766		func (s *Service) recoveryInhibited(ctx context.Context, app, service, containerID string) (bool, error) {	866		func (s *Service) recoveryInhibited(ctx context.Context, app, service, containerID string) (bool, error) {
767		if containerID == "" {	867		if containerID == "" {
768		return false, nil	868		return false, nil

@@ -772,7 +872,8 @@ func (s *Service) recoveryInhibited(ctx context.Context, app, service, container
772		return false, fmt.Errorf("deployment: load recovery inhibitions: %w", err)	872		return false, fmt.Errorf("deployment: load recovery inhibitions: %w", err)
773		}	873		}
774		for _, inhibition := range inhibitions {	874		for _, inhibition := range inhibitions {
775	-	if inhibition.Service == service && inhibition.ContainerID == containerID {	875	+	if inhibition.Service == service && inhibition.ContainerID == containerID &&
			876	+	inhibition.Reason != domain.AppInhibitRetirementPending {
776		return true, nil	877		return true, nil
777		}	878		}
778		}	879		}

@@ -799,6 +900,28 @@ func (s *Service) inhibitRecovery(ctx context.Context, app, service, containerID
799		return nil	900		return nil
800		}	901		}
801			902
			903	+	// inhibitRetirement durably marks a superseded container that an overlap
			904	+	// replacement left running beside its successor. The periodic reconciliation
			905	+	// retires it with grace, once the successor is verified and routed.
			906	+	func (s *Service) inhibitRetirement(ctx context.Context, app, service, containerID, operation string, grace time.Duration) error {
			907	+	if containerID == "" {
			908	+	return nil
			909	+	}
			910	+	inhibition := domain.AppRecoveryInhibition{
			911	+	App: app,
			912	+	Service: service,
			913	+	ContainerID: containerID,
			914	+	Reason: domain.AppInhibitRetirementPending,
			915	+	Operation: operation,
			916	+	CreatedAt: time.Now().UTC(),
			917	+	StopGrace: grace,
			918	+	}
			919	+	if err := s.deps.State.SaveRecoveryInhibition(ctx, inhibition); err != nil {
			920	+	return fmt.Errorf("deployment: record retirement of %s/%s: %w", app, containerID, err)
			921	+	}
			922	+	return nil
			923	+	}
			924	+
802		// clearRecoveryInhibition drops the inhibition of a safely superseded	925		// clearRecoveryInhibition drops the inhibition of a safely superseded
803		// generation.	926		// generation.
804		func (s *Service) clearRecoveryInhibition(ctx context.Context, app, service, containerID string) error {	927		func (s *Service) clearRecoveryInhibition(ctx context.Context, app, service, containerID string) error {

M internal/usecase/deployment/networks_internal_test.go +4 -0


@@ -48,6 +48,7 @@ func TestCreateAndStart_UsesIncarnationPrivateNetworkAndLimits(t *testing.T) {
48	48		assert.Equal(t, "web", cfg.Hostname)
49	49		})).Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
50	50		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
	51	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
51	52
52	53		svc := networkTestService(t, state, runtime,
53	54		NetworkConfig{Prefix: "gordon", Internal: true},

@@ -77,6 +78,7 @@ func TestCreateAndStart_ReusesOwnedNetwork(t *testing.T) {
77	78		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).
78	79		Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
79	80		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
	81	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
80	82
81	83		svc := networkTestService(t, state, runtime, NetworkConfig{Prefix: "gordon"}, ResourceLimits{})
82	84		_, err := svc.createAndStart(ctx, "blog", "rev-1",

@@ -124,6 +126,7 @@ func TestCreateAndStart_JoinsDeclaredSharedNetworks(t *testing.T) {
124	126		Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
125	127		runtime.EXPECT().ConnectContainerToNetwork(mock.Anything, "c-1", shared).Return(nil).Once()
126	128		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
	129	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
127	130
128	131		svc := networkTestService(t, state, runtime, NetworkConfig{Prefix: "gordon"}, ResourceLimits{})
129	132		_, err := svc.createAndStart(ctx, "blog", "rev-1", pinnedService{

@@ -155,6 +158,7 @@ func TestCreateAndStart_RoutesDeclaredReadOnlyVolumeToReadOnlyMount(t *testing.T
155	158		return readOnly && !writable && cfg.ReadOnlyVolumes["/config"] == runtimeName
156	159		})).Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
157	160		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
	161	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
158	162
159	163		svc := networkTestService(t, state, runtime, NetworkConfig{Prefix: "gordon"}, ResourceLimits{})
160	164		_, err := svc.createAndStart(ctx, "blog", "rev-1", pinnedService{


@@ -48,6 +48,7 @@ func TestCreateAndStart_UsesIncarnationPrivateNetworkAndLimits(t *testing.T) {
48	assert.Equal(t, "web", cfg.Hostname)	48		assert.Equal(t, "web", cfg.Hostname)
49	})).Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()	49		})).Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
50	runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()	50		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
		51	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
51		52
52	svc := networkTestService(t, state, runtime,	53		svc := networkTestService(t, state, runtime,
53	NetworkConfig{Prefix: "gordon", Internal: true},	54		NetworkConfig{Prefix: "gordon", Internal: true},

@@ -77,6 +78,7 @@ func TestCreateAndStart_ReusesOwnedNetwork(t *testing.T) {
77	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).	78		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).
78	Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()	79		Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
79	runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()	80		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
		81	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
80		82
81	svc := networkTestService(t, state, runtime, NetworkConfig{Prefix: "gordon"}, ResourceLimits{})	83		svc := networkTestService(t, state, runtime, NetworkConfig{Prefix: "gordon"}, ResourceLimits{})
82	_, err := svc.createAndStart(ctx, "blog", "rev-1",	84		_, err := svc.createAndStart(ctx, "blog", "rev-1",

@@ -124,6 +126,7 @@ func TestCreateAndStart_JoinsDeclaredSharedNetworks(t *testing.T) {
124	Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()	126		Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
125	runtime.EXPECT().ConnectContainerToNetwork(mock.Anything, "c-1", shared).Return(nil).Once()	127		runtime.EXPECT().ConnectContainerToNetwork(mock.Anything, "c-1", shared).Return(nil).Once()
126	runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()	128		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
		129	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
127		130
128	svc := networkTestService(t, state, runtime, NetworkConfig{Prefix: "gordon"}, ResourceLimits{})	131		svc := networkTestService(t, state, runtime, NetworkConfig{Prefix: "gordon"}, ResourceLimits{})
129	_, err := svc.createAndStart(ctx, "blog", "rev-1", pinnedService{	132		_, err := svc.createAndStart(ctx, "blog", "rev-1", pinnedService{

@@ -155,6 +158,7 @@ func TestCreateAndStart_RoutesDeclaredReadOnlyVolumeToReadOnlyMount(t *testing.T
155	return readOnly && !writable && cfg.ReadOnlyVolumes["/config"] == runtimeName	158		return readOnly && !writable && cfg.ReadOnlyVolumes["/config"] == runtimeName
156	})).Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()	159		})).Return(&domain.Container{ID: "c-1", Name: "web"}, nil).Once()
157	runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()	160		runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
		161	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
158		162
159	svc := networkTestService(t, state, runtime, NetworkConfig{Prefix: "gordon"}, ResourceLimits{})	163		svc := networkTestService(t, state, runtime, NetworkConfig{Prefix: "gordon"}, ResourceLimits{})
160	_, err := svc.createAndStart(ctx, "blog", "rev-1", pinnedService{	164		_, err := svc.createAndStart(ctx, "blog", "rev-1", pinnedService{

A internal/usecase/deployment/overlap_replacement_test.go +1016 -0

@@ -0,0 +1,1016 @@
	1	+	package deployment_test
	2	+
	3	+	import (
	4	+	"context"
	5	+	"maps"
	6	+	"os"
	7	+	"path/filepath"
	8	+	"sync"
	9	+	"testing"
	10	+	"time"
	11	+
	12	+	"github.com/bnema/zerowrap"
	13	+	"github.com/stretchr/testify/assert"
	14	+	"github.com/stretchr/testify/mock"
	15	+	"github.com/stretchr/testify/require"
	16	+
	17	+	"github.com/bnema/gordon/internal/adapters/out/appstate"
	18	+	outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
	19	+	"github.com/bnema/gordon/internal/domain"
	20	+	"github.com/bnema/gordon/internal/usecase/deployment"
	21	+	)
	22	+
	23	+	// TestDeploy_NonOverlapEligibleServicesKeepSequentialOrder proves a service
	24	+	// with a volume, a bind, or a device is never run beside its predecessor: the
	25	+	// service is withdrawn, the superseded container is stopped and removed, and
	26	+	// only then is the replacement created.
	27	+	func TestDeploy_NonOverlapEligibleServicesKeepSequentialOrder(t *testing.T) {
	28	+	root := t.TempDir()
	29	+	source := filepath.Join(root, "binds", "config")
	30	+	require.NoError(t, os.MkdirAll(source, 0o755))
	31	+
	32	+	cases := []struct {
	33	+	name string
	34	+	mutate func(*domain.AppService)
	35	+	inhibits bool
	36	+	cdiSupported bool
	37	+	}{
	38	+	{"volume", func(s *domain.AppService) { s.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}} }, true, false},
	39	+	{"bind", func(s *domain.AppService) {
	40	+	s.Binds = []domain.AppBind{{Name: "config", Path: "/etc/app.conf", ReadOnly: true}}
	41	+	}, true, false},
	42	+	{"device", func(s *domain.AppService) { s.Devices = []string{"test_gpu"} }, false, true},
	43	+	{"udp", func(s *domain.AppService) {
	44	+	s.UDP = []domain.AppUDPInterface{{Entrypoint: "game", Port: 28015, Publish: "0.0.0.0:28015"}}
	45	+	}, false, false},
	46	+	}
	47	+	for _, tc := range cases {
	48	+	t.Run(tc.name, func(t *testing.T) {
	49	+	ctx := context.Background()
	50	+	state, runtime, images, secrets := mockDeps(t)
	51	+	rev := mockRevision()
	52	+	tc.mutate(&rev.Spec.Services[0])
	53	+
	54	+	expectDeployPreflight(state, runtime, images, secrets, rev)
	55	+	runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil).Maybe()
	56	+	runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe()
	57	+	if tc.cdiSupported {
	58	+	runtime.EXPECT().SupportsCDIDevices(mock.Anything).Return(nil)
	59	+	}
	60	+	expectNetworkProvision(runtime, "app-blog", 1)
	61	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
	62	+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	63	+	if tc.inhibits {
	64	+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
	65	+	}
	66	+
	67	+	var order []string
	68	+	expectRetireOld(state, runtime, &order, "c-old")
	69	+	expectSequentialCreate(state, runtime, &order, "c-new", 18080)
	70	+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(
	71	+	func(context.Context, domain.AppActive) error {
	72	+	recordEvent(&order, "publish-active")
	73	+	return nil
	74	+	}).Once()
	75	+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
	76	+	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
	77	+
	78	+	svc := deployment.NewService(deployment.Deps{
	79	+	State: state, Runtime: runtime, Images: images, Secrets: secrets,
	80	+	Traffic: &orderedTraffic{events: &order},
	81	+	}, zerowrap.Default()).
	82	+	WithBindPolicies(map[string]domain.AppBindPolicy{"config": {
	83	+	Name: "config", Source: source, Root: root, AllowedApps: []string{"blog"}, AllowedServices: []string{"web"},
	84	+	}}).
	85	+	WithDevicePolicies(map[string]domain.AppDevicePolicy{"test_gpu": {
	86	+	Name: "test_gpu", CDI: []string{"example.com/gpu=GPU-test-uuid"},
	87	+	AllowedApps: []string{"blog"}, AllowedServices: []string{"web"},
	88	+	}}).
	89	+	WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	90	+	func(context.Context, string, string) (int, error) { return 200, nil },
	91	+	func(context.Context, string) error { return nil },
	92	+	))
	93	+
	94	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	95	+	require.NoError(t, err)
	96	+	assert.Equal(t, "deployed", result.Services["web"].Result)
	97	+	assert.Equal(t,
	98	+	[]string{"withdraw", "stop-old", "remove-old", "create-new", "start-new", "publish-active", "traffic"},
	99	+	order, "the superseded container is gone before the replacement is created")
	100	+	})
	101	+	}
	102	+	}
	103	+
	104	+	// TestDeploy_OverlapPublicationFailureKeepsOldContainer proves a failed ACTIVE
	105	+	// publication of an overlap replacement never stops the superseded container,
	106	+	// which still serves because ACTIVE and routing were not switched. The
	107	+	// candidate stays journaled as the step's After, so reconciliation removes it,
	108	+	// and the failure is reported with a leftover warning on the old container.
	109	+	func TestDeploy_OverlapPublicationFailureKeepsOldContainer(t *testing.T) {
	110	+	ctx := context.Background()
	111	+	state, runtime, images, secrets := mockDeps(t)
	112	+	rev := mockRevision()
	113	+
	114	+	expectDeployPreflight(state, runtime, images, secrets, rev)
	115	+	expectNetworkProvision(runtime, "app-blog", 1)
	116	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
	117	+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	118	+	expectSequentialCreate(state, runtime, nil, "c-new", 18080)
	119	+	expectRetirementMarker(state, "c-old")
	120	+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(assert.AnError).Once()
	121	+
	122	+	var events []string
	123	+	svc := deployment.NewService(deployment.Deps{
	124	+	State: state, Runtime: runtime, Images: images, Secrets: secrets,
	125	+	Traffic: &orderedTraffic{events: &events},
	126	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	127	+	func(context.Context, string, string) (int, error) { return 200, nil },
	128	+	func(context.Context, string) error { return nil },
	129	+	))
	130	+
	131	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	132	+	require.Error(t, err)
	133	+	assert.Equal(t, "failed", result.Services["web"].Result)
	134	+	assert.Equal(t, "c-new", result.Services["web"].After)
	135	+	require.Len(t, result.Services["web"].CleanupWarnings, 1)
	136	+	assert.Equal(t, "c-old", result.Services["web"].CleanupWarnings[0].Leftover)
	137	+	assert.Empty(t, events, "the proxy graph was neither withdrawn nor rebuilt")
	138	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
	139	+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
	140	+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
	141	+	}
	142	+
	143	+	// storeTraffic is a traffic publisher that behaves like the real one against
	144	+	// a real app-state store: a withdrawal drops the service's recorded binds from
	145	+	// ACTIVE (and can then report a failed graph apply), and a rebuild routes
	146	+	// exactly the binds ACTIVE records at that moment. Tests assert on what is
	147	+	// routed, not on call order.
	148	+	type storeTraffic struct {
	149	+	mu sync.Mutex
	150	+	store *appstate.Store
	151	+	failWithdraw int
	152	+	failRebuild int
	153	+	withdrawn []string
	154	+	routed []map[string]map[int]int
	155	+	}
	156	+
	157	+	func (t *storeTraffic) WithdrawService(ctx context.Context, app, service string) error {
	158	+	t.mu.Lock()
	159	+	defer t.mu.Unlock()
	160	+	t.withdrawn = append(t.withdrawn, app+"/"+service)
	161	+	active, ok, err := t.store.LoadActive(ctx, app)
	162	+	if err != nil \|\| !ok {
	163	+	return err
	164	+	}
	165	+	if eff, found := active.Services[service]; found {
	166	+	eff.BackendBinds, eff.UDPBackendBinds = nil, nil
	167	+	active.Services[service] = eff
	168	+	if err := t.store.SaveActive(ctx, active); err != nil {
	169	+	return err
	170	+	}
	171	+	}
	172	+	if t.failWithdraw > 0 {
	173	+	t.failWithdraw--
	174	+	return assert.AnError
	175	+	}
	176	+	return nil
	177	+	}
	178	+
	179	+	func (t *storeTraffic) WithdrawServiceState(ctx context.Context, app, service string) error {
	180	+	return t.WithdrawService(ctx, app, service)
	181	+	}
	182	+
	183	+	func (t *storeTraffic) RebuildTraffic(ctx context.Context) error {
	184	+	t.mu.Lock()
	185	+	defer t.mu.Unlock()
	186	+	if t.failRebuild > 0 {
	187	+	t.failRebuild--
	188	+	return assert.AnError
	189	+	}
	190	+	active, _, err := t.store.LoadActive(ctx, "blog")
	191	+	if err != nil {
	192	+	return err
	193	+	}
	194	+	routes := map[string]map[int]int{}
	195	+	for name, eff := range active.Services {
	196	+	routes[name] = maps.Clone(eff.BackendBinds)
	197	+	}
	198	+	t.routed = append(t.routed, routes)
	199	+	return nil
	200	+	}
	201	+
	202	+	// lastRoute returns the binds the most recent graph routed for a service.
	203	+	func (t *storeTraffic) lastRoute(service string) map[int]int {
	204	+	t.mu.Lock()
	205	+	defer t.mu.Unlock()
	206	+	if len(t.routed) == 0 {
	207	+	return nil
	208	+	}
	209	+	return t.routed[len(t.routed)-1][service]
	210	+	}
	211	+
	212	+	// allRoutes returns the binds of every graph routed for a service.
	213	+	func (t *storeTraffic) allRoutes(service string) []map[int]int {
	214	+	t.mu.Lock()
	215	+	defer t.mu.Unlock()
	216	+	var routes []map[int]int
	217	+	for _, routed := range t.routed {
	218	+	routes = append(routes, routed[service])
	219	+	}
	220	+	return routes
	221	+	}
	222	+
	223	+	// withdrawals returns the services withdrawn so far, in order.
	224	+	func (t *storeTraffic) withdrawals() []string {
	225	+	t.mu.Lock()
	226	+	defer t.mu.Unlock()
	227	+	return append([]string(nil), t.withdrawn...)
	228	+	}
	229	+
	230	+	func (t *storeTraffic) rebuildCount() int {
	231	+	t.mu.Lock()
	232	+	defer t.mu.Unlock()
	233	+	return len(t.routed)
	234	+	}
	235	+
	236	+	const overlapImage = "registry.example.com/blog/web:1.4.2"
	237	+
	238	+	// overlapWebSpec is a stateless public HTTP service.
	239	+	func overlapWebSpec(grace time.Duration) domain.AppService {
	240	+	return domain.AppService{
	241	+	Name: "web", Image: overlapImage, Secrets: map[string]string{}, StopGrace: grace,
	242	+	HTTP: []domain.AppHTTPInterface{{Host: "blog.example.com", Port: 8080, TLS: domain.AppTLSAuto}},
	243	+	Readiness: domain.AppReadiness{
	244	+	Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: time.Second,
	245	+	},
	246	+	}
	247	+	}
	248	+
	249	+	func overlapOldBinds() map[int]int { return map[int]int{8080: 17000} }
	250	+
	251	+	// seedServingApp seeds an app whose container c-old serves on 17000, with a
	252	+	// materialized revision to deploy over it.
	253	+	func seedServingApp(t testing.T, ctx context.Context, store appstate.Store, spec domain.AppService) {
	254	+	t.Helper()
	255	+	require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
	256	+	require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
	257	+	require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	258	+	"web": {Container: "c-old", EffectiveRevision: "rev-0", Spec: spec, BackendBinds: overlapOldBinds()},
	259	+	}}))
	260	+	seedDesired(t, ctx, store, testRevision("blog", spec))
	261	+	}
	262	+
	263	+	// overlapDeployDeps wires the preflight and candidate creation of one overlap
	264	+	// deploy that creates c-new on 18081 and expects readiness to pass.
	265	+	func overlapDeployMocks(runtime outmocks.MockContainerRuntime, images outmocks.MockImageResolver) {
	266	+	images.EXPECT().ResolveDigest(mock.Anything, overlapImage).Return(restartTestDigest, nil).Once()
	267	+	runtime.EXPECT().InspectImageVolumes(mock.Anything, overlapImage).Return(nil, nil).Once()
	268	+	expectNetworkProvision(runtime, "app-blog", 1)
	269	+	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
	270	+	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	271	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
	272	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	273	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
	274	+	}
	275	+
	276	+	func overlapService(store appstate.Store, runtime outmocks.MockContainerRuntime, images outmocks.MockImageResolver, secrets outmocks.MockSecretProvider, traffic storeTraffic) deployment.Service {
	277	+	return deployment.NewService(deployment.Deps{
	278	+	State: store, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
	279	+	ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
	280	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	281	+	func(context.Context, string, string) (int, error) { return 200, nil },
	282	+	func(context.Context, string) error { return nil },
	283	+	))
	284	+	}
	285	+
	286	+	// expectVerifiedRunning wires one recovery verification of a running
	287	+	// container that publishes the given loopback port.
	288	+	func expectVerifiedRunning(runtime *outmocks.MockContainerRuntime, id string, port int) {
	289	+	started := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
	290	+	runtime.EXPECT().InspectContainer(mock.Anything, id).
	291	+	Return(&domain.Container{ID: id, Status: "running", StartedAt: started}, nil)
	292	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, id).Return("", false, nil)
	293	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, id, mock.Anything).Return(
	294	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: port, Protocol: domain.NetworkProtocolTCP}}, nil)
	295	+	}
	296	+
	297	+	// TestDeploy_OverlapSwitchIsNotErasedByAPendingWithdrawal proves that a
	298	+	// publication withdrawal left pending by an earlier failed recovery does not
	299	+	// unroute the new generation: the deploy resolves it before publishing, so
	300	+	// the graph the switch applies routes the candidate's binds. The traffic fake
	301	+	// mutates ACTIVE like the real publisher, so a stray withdrawal after the
	302	+	// switch would leave the service unrouted.
	303	+	func TestDeploy_OverlapSwitchIsNotErasedByAPendingWithdrawal(t *testing.T) {
	304	+	ctx := context.Background()
	305	+	store := newTestStore(t)
	306	+	runtime := outmocks.NewMockContainerRuntime(t)
	307	+	images := outmocks.NewMockImageResolver(t)
	308	+	secrets := outmocks.NewMockSecretProvider(t)
	309	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	310	+	traffic := &storeTraffic{store: store, failWithdraw: 1}
	311	+	svc := overlapService(store, runtime, images, secrets, traffic)
	312	+
	313	+	// A recovery pass finds c-old down and cannot prove its withdrawal: the
	314	+	// service is marked as having a pending withdrawal.
	315	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").
	316	+	Return(&domain.Container{ID: "c-old", Status: "exited"}, nil).Once()
	317	+	require.Error(t, svc.ReconcileRunning(ctx))
	318	+
	319	+	overlapDeployMocks(runtime, images)
	320	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", domain.AppDefaultStopGrace).Return(nil).Once()
	321	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	322	+
	323	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	324	+	require.NoError(t, err)
	325	+	assert.Equal(t, "deployed", result.Services["web"].Result)
	326	+
	327	+	assert.Equal(t, map[int]int{8080: 18081}, traffic.lastRoute("web"), "the service routes the new generation")
	328	+	active, _, err := store.LoadActive(ctx, "blog")
	329	+	require.NoError(t, err)
	330	+	assert.Equal(t, "c-new", active.Services["web"].Container)
	331	+	assert.Equal(t, map[int]int{8080: 18081}, active.Services["web"].BackendBinds,
	332	+	"ACTIVE keeps only the new generation's binds")
	333	+	}
	334	+
	335	+	// TestDeploy_OverlapRetirementSurvivesANewerOperation proves the cleanup of a
	336	+	// superseded container is owned by the durable retirement marker, not by the
	337	+	// operation journal: after the first retirement fails and a newer operation
	338	+	// becomes the latest journal, the periodic reconciliation still retires it,
	339	+	// with the predecessor's own stop grace, and only once its successor is
	340	+	// verified and routed.
	341	+	func TestDeploy_OverlapRetirementSurvivesANewerOperation(t *testing.T) {
	342	+	ctx := context.Background()
	343	+	store := newTestStore(t)
	344	+	runtime := outmocks.NewMockContainerRuntime(t)
	345	+	images := outmocks.NewMockImageResolver(t)
	346	+	secrets := outmocks.NewMockSecretProvider(t)
	347	+	seedServingApp(t, ctx, store, overlapWebSpec(45*time.Second))
	348	+	traffic := &storeTraffic{store: store}
	349	+	svc := overlapService(store, runtime, images, secrets, traffic)
	350	+
	351	+	overlapDeployMocks(runtime, images)
	352	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", 45*time.Second).Return(nil).Twice()
	353	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
	354	+
	355	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	356	+	require.NoError(t, err, "a retirement leftover does not fail a deploy whose new generation serves")
	357	+	require.Len(t, result.CleanupWarnings, 1)
	358	+	assert.Equal(t, "c-old", result.CleanupWarnings[0].Leftover)
	359	+
	360	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	361	+	require.NoError(t, err)
	362	+	require.Len(t, markers, 1, "the failed retirement leaves its durable marker")
	363	+	assert.Equal(t, "c-old", markers[0].ContainerID)
	364	+	assert.Equal(t, domain.AppInhibitRetirementPending, markers[0].Reason)
	365	+	assert.Equal(t, 45*time.Second, markers[0].StopGrace, "the predecessor's grace travels with the marker")
	366	+
	367	+	// A newer operation becomes the latest journal and masks the deploy's.
	368	+	require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
	369	+	Op: "op-newer", Kind: "deploy", App: "blog", StartedAt: time.Now().Add(time.Minute).UTC(),
	370	+	Outcome: domain.AppOutcomeSuccess,
	371	+	}))
	372	+
	373	+	expectVerifiedRunning(runtime, "c-new", 18082)
	374	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	375	+	require.NoError(t, svc.ReconcileRunning(ctx))
	376	+
	377	+	markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
	378	+	require.NoError(t, err)
	379	+	assert.Empty(t, markers, "the marker is cleared once the container is confirmed gone")
	380	+	assert.Equal(t, map[int]int{8080: 18082}, traffic.lastRoute("web"), "the verified successor is routed")
	381	+	active, _, err := store.LoadActive(ctx, "blog")
	382	+	require.NoError(t, err)
	383	+	assert.Equal(t, "c-new", active.Services["web"].Container)
	384	+	}
	385	+
	386	+	// TestReconcileRunning_RetiresSupersededOnlyAfterSuccessorIsVerified proves a
	387	+	// superseded container is kept while its successor cannot be verified: the
	388	+	// working generation is never stopped in favour of one that is not serving.
	389	+	func TestReconcileRunning_RetiresSupersededOnlyAfterSuccessorIsVerified(t *testing.T) {
	390	+	ctx := context.Background()
	391	+	store := newTestStore(t)
	392	+	runtime := outmocks.NewMockContainerRuntime(t)
	393	+	images := outmocks.NewMockImageResolver(t)
	394	+	secrets := outmocks.NewMockSecretProvider(t)
	395	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	396	+	require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	397	+	"web": {Container: "c-new", EffectiveRevision: "rev-1", Spec: overlapWebSpec(0), BackendBinds: map[int]int{8080: 18081}},
	398	+	}}))
	399	+	require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
	400	+	App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitRetirementPending,
	401	+	}))
	402	+	traffic := &storeTraffic{store: store}
	403	+	svc := overlapService(store, runtime, images, secrets, traffic)
	404	+
	405	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(nil, assert.AnError).Once()
	406	+	require.Error(t, svc.ReconcileRunning(ctx))
	407	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
	408	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	409	+	require.NoError(t, err)
	410	+	assert.Len(t, markers, 1, "the marker waits for a verified successor")
	411	+
	412	+	// A record written before the grace was persisted falls back to the default.
	413	+	expectVerifiedRunning(runtime, "c-new", 18081)
	414	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", domain.AppDefaultStopGrace).Return(nil).Once()
	415	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	416	+	require.NoError(t, svc.ReconcileRunning(ctx))
	417	+	markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
	418	+	require.NoError(t, err)
	419	+	assert.Empty(t, markers)
	420	+	}
	421	+
	422	+	// TestReconcileRunning_NeverRetiresAPendingContainerThatActiveNamesAgain
	423	+	// proves a rollback to the superseded container is safe: when ACTIVE names the
	424	+	// container again, its stale retirement marker is dropped and it is kept.
	425	+	func TestReconcileRunning_NeverRetiresAPendingContainerThatActiveNamesAgain(t *testing.T) {
	426	+	ctx := context.Background()
	427	+	store := newTestStore(t)
	428	+	runtime := outmocks.NewMockContainerRuntime(t)
	429	+	images := outmocks.NewMockImageResolver(t)
	430	+	secrets := outmocks.NewMockSecretProvider(t)
	431	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	432	+	require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
	433	+	App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitRetirementPending,
	434	+	}))
	435	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	436	+
	437	+	expectVerifiedRunning(runtime, "c-old", 17000)
	438	+	require.NoError(t, svc.ReconcileRunning(ctx))
	439	+
	440	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
	441	+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
	442	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	443	+	require.NoError(t, err)
	444	+	assert.Empty(t, markers)
	445	+	active, _, err := store.LoadActive(ctx, "blog")
	446	+	require.NoError(t, err)
	447	+	assert.Equal(t, "c-old", active.Services["web"].Container)
	448	+	}
	449	+
	450	+	// TestBootRecovery_DoesNotRepublishUnverifiedBindsOfAPublishedCandidate proves
	451	+	// interrupted-operation reconciliation never rebuilds traffic from the binds
	452	+	// ACTIVE persisted for a published candidate: a restarted candidate's
	453	+	// ephemeral port may belong to another workload. Only the verified start path
	454	+	// routes, and the superseded container is retired afterwards through its
	455	+	// marker.
	456	+	func TestBootRecovery_DoesNotRepublishUnverifiedBindsOfAPublishedCandidate(t *testing.T) {
	457	+	ctx := context.Background()
	458	+	store := newTestStore(t)
	459	+	runtime := outmocks.NewMockContainerRuntime(t)
	460	+	images := outmocks.NewMockImageResolver(t)
	461	+	secrets := outmocks.NewMockSecretProvider(t)
	462	+	spec := overlapWebSpec(0)
	463	+	require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
	464	+	require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	465	+	"web": {Container: "c-new", EffectiveRevision: "rev-1", Spec: spec, BackendBinds: map[int]int{8080: 18081}},
	466	+	}}))
	467	+	require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
	468	+	Op: "op-switch", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
	469	+	Steps: []domain.AppOperationStep{
	470	+	{ID: "preflight", State: domain.AppStepSucceeded},
	471	+	{ID: "service.web.replace", State: domain.AppStepPending, Service: "web", Before: "c-old", After: "c-new"},
	472	+	},
	473	+	}))
	474	+	require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
	475	+	App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitRetirementPending,
	476	+	}))
	477	+	traffic := &storeTraffic{store: store}
	478	+	svc := overlapService(store, runtime, images, secrets, traffic)
	479	+
	480	+	// The candidate restarted and now publishes a different port.
	481	+	runtime.EXPECT().IsContainerRunning(mock.Anything, "c-new").Return(true, nil).Once()
	482	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
	483	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	484	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18090, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
	485	+	require.NoError(t, svc.ReconcileBoot(ctx))
	486	+
	487	+	require.Positive(t, traffic.rebuildCount())
	488	+	for _, routed := range traffic.allRoutes("web") {
	489	+	assert.NotEqual(t, map[int]int{8080: 18081}, routed, "the unverified persisted port is never routed")
	490	+	}
	491	+	assert.Equal(t, map[int]int{8080: 18090}, traffic.lastRoute("web"), "the verified bind is routed")
	492	+	active, _, err := store.LoadActive(ctx, "blog")
	493	+	require.NoError(t, err)
	494	+	assert.Equal(t, map[int]int{8080: 18090}, active.Services["web"].BackendBinds)
	495	+
	496	+	// Boot keeps the superseded container and its marker: the periodic pass
	497	+	// retires it once the successor is verified.
	498	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
	499	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	500	+	require.NoError(t, err)
	501	+	require.Len(t, markers, 1)
	502	+	assert.Equal(t, "c-old", markers[0].ContainerID)
	503	+	}
	504	+
	505	+	// TestDeploy_OverlapTrafficFailureIsRecoveredByThePeriodicPass proves the
	506	+	// failed switch is recoverable and traceable: the graph apply is rejected after
	507	+	// ACTIVE named the new container, the superseded container keeps running with
	508	+	// its marker, the next deploy-side reconciliation keeps that marker, and the
	509	+	// periodic pass retires the old container once the successor is routed.
	510	+	func TestDeploy_OverlapTrafficFailureIsRecoveredByThePeriodicPass(t *testing.T) {
	511	+	ctx := context.Background()
	512	+	store := newTestStore(t)
	513	+	runtime := outmocks.NewMockContainerRuntime(t)
	514	+	images := outmocks.NewMockImageResolver(t)
	515	+	secrets := outmocks.NewMockSecretProvider(t)
	516	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	517	+	traffic := &storeTraffic{store: store, failRebuild: 1}
	518	+	svc := overlapService(store, runtime, images, secrets, traffic)
	519	+
	520	+	overlapDeployMocks(runtime, images)
	521	+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	522	+	require.Error(t, err, "a rejected graph is a failed deploy")
	523	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
	524	+
	525	+	active, _, err := store.LoadActive(ctx, "blog")
	526	+	require.NoError(t, err)
	527	+	assert.Equal(t, "c-new", active.Services["web"].Container, "ACTIVE names the new container")
	528	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	529	+	require.NoError(t, err)
	530	+	require.Len(t, markers, 1)
	531	+	assert.Equal(t, "c-old", markers[0].ContainerID)
	532	+
	533	+	// The next mutation reconciles the failed deploy: the published candidate
	534	+	// is kept, and so is the retirement marker.
	535	+	require.NoError(t, svc.AbandonDeploy(ctx, deployment.DeployClaim{App: "blog", Op: "op-none"}))
	536	+	markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
	537	+	require.NoError(t, err)
	538	+	require.Len(t, markers, 1, "a published candidate's reconciliation keeps the retirement marker")
	539	+
	540	+	expectVerifiedRunning(runtime, "c-new", 18081)
	541	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", domain.AppDefaultStopGrace).Return(nil).Once()
	542	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	543	+	require.NoError(t, svc.ReconcileRunning(ctx))
	544	+	assert.Equal(t, map[int]int{8080: 18081}, traffic.lastRoute("web"))
	545	+	markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
	546	+	require.NoError(t, err)
	547	+	assert.Empty(t, markers)
	548	+	}
	549	+
	550	+	// TestDeploy_PredecessorWithOwnedResourcesNeverOverlaps proves eligibility
	551	+	// looks at the predecessor's effective spec too: a replacement that drops the
	552	+	// UDP interface or the volume of the generation it supersedes still replaces
	553	+	// it sequentially, because the old generation's UDP sessions or writes
	554	+	// outlive the switch.
	555	+	func TestDeploy_PredecessorWithOwnedResourcesNeverOverlaps(t *testing.T) {
	556	+	cases := map[string]func(*domain.AppService){
	557	+	"udp": func(s *domain.AppService) {
	558	+	s.UDP = []domain.AppUDPInterface{{Entrypoint: "game", Port: 28015, Publish: "0.0.0.0:28015"}}
	559	+	},
	560	+	"volume": func(s *domain.AppService) { s.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}} },
	561	+	}
	562	+	for name, mutate := range cases {
	563	+	t.Run(name, func(t *testing.T) {
	564	+	ctx := context.Background()
	565	+	state, runtime, images, secrets := mockDeps(t)
	566	+	rev := mockRevision() // HTTP-only, no volume: eligible on its own
	567	+	previous := rev.Spec.Services[0]
	568	+	mutate(&previous)
	569	+	active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	570	+	"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", Spec: previous},
	571	+	}}
	572	+
	573	+	expectDeployPreflight(state, runtime, images, secrets, rev)
	574	+	expectNetworkProvision(runtime, "app-blog", 1)
	575	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
	576	+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	577	+	var order []string
	578	+	expectRetireOld(state, runtime, &order, "c-old")
	579	+	expectSequentialCreate(state, runtime, &order, "c-new", 18080)
	580	+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
	581	+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
	582	+	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
	583	+
	584	+	svc := deployment.NewService(deployment.Deps{
	585	+	State: state, Runtime: runtime, Images: images, Secrets: secrets,
	586	+	Traffic: &orderedTraffic{events: &order},
	587	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	588	+	func(context.Context, string, string) (int, error) { return 200, nil },
	589	+	func(context.Context, string) error { return nil },
	590	+	))
	591	+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	592	+	require.NoError(t, err)
	593	+	assert.Equal(t, []string{"withdraw", "stop-old", "remove-old", "create-new", "start-new", "traffic"}, order[:6],
	594	+	"the predecessor is gone before the replacement is created")
	595	+	state.AssertNotCalled(t, "SaveRecoveryInhibition", mock.Anything, mock.Anything)
	596	+	})
	597	+	}
	598	+	}
	599	+
	600	+	// seedPendingRetirement seeds ACTIVE naming c-new and one retirement-pending
	601	+	// marker per given superseded container.
	602	+	func seedPendingRetirement(t testing.T, ctx context.Context, store appstate.Store, superseded ...string) {
	603	+	t.Helper()
	604	+	require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
	605	+	require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	606	+	"web": {Container: "c-new", EffectiveRevision: "rev-1", Spec: overlapWebSpec(0), BackendBinds: map[int]int{8080: 18081}},
	607	+	}}))
	608	+	for _, id := range superseded {
	609	+	require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
	610	+	App: "blog", Service: "web", ContainerID: id, Reason: domain.AppInhibitRetirementPending,
	611	+	}))
	612	+	}
	613	+	}
	614	+
	615	+	func markerIDs(t testing.T, ctx context.Context, store appstate.Store) []string {
	616	+	t.Helper()
	617	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	618	+	require.NoError(t, err)
	619	+	ids := make([]string, 0, len(markers))
	620	+	for _, marker := range markers {
	621	+	ids = append(ids, marker.ContainerID)
	622	+	}
	623	+	return ids
	624	+	}
	625	+
	626	+	// TestReconcileRunning_DoesNotRetireWhenSuccessorIsNotVerifiedByThisPass is the
	627	+	// regression for a successor that was verified by an earlier pass but is
	628	+	// restarting in this one: it is withdrawn, not verified, and a final
	629	+	// inspection that happens to see it running must not retire the predecessor.
	630	+	func TestReconcileRunning_DoesNotRetireWhenSuccessorIsNotVerifiedByThisPass(t *testing.T) {
	631	+	ctx := context.Background()
	632	+	store := newTestStore(t)
	633	+	runtime := outmocks.NewMockContainerRuntime(t)
	634	+	images := outmocks.NewMockImageResolver(t)
	635	+	secrets := outmocks.NewMockSecretProvider(t)
	636	+	seedPendingRetirement(t, ctx, store, "c-old")
	637	+	traffic := &storeTraffic{store: store}
	638	+	svc := overlapService(store, runtime, images, secrets, traffic)
	639	+
	640	+	// Earlier pass: the successor is verified and routed, retirement fails.
	641	+	expectVerifiedRunning(runtime, "c-new", 18081)
	642	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	643	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
	644	+	require.Error(t, svc.ReconcileRunning(ctx))
	645	+
	646	+	// This pass: the successor is restarting, so it is withdrawn and not
	647	+	// verified. Any later inspection would see it running.
	648	+	runtime.ExpectedCalls, runtime.Calls = nil, nil
	649	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").
	650	+	Return(&domain.Container{ID: "c-new", Status: "restarting"}, nil).Once()
	651	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").
	652	+	Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: time.Now()}, nil).Maybe()
	653	+	require.NoError(t, svc.ReconcileRunning(ctx))
	654	+
	655	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
	656	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "the predecessor stays until a pass verifies the successor")
	657	+	}
	658	+
	659	+	// TestReconcileRunning_DoesNotRetireWhenSuccessorRestartedAfterVerification
	660	+	// covers the other half: a successor verified at one start time and seen with
	661	+	// another by the final inspection is a different execution, so unverified.
	662	+	func TestReconcileRunning_DoesNotRetireWhenSuccessorRestartedAfterVerification(t *testing.T) {
	663	+	ctx := context.Background()
	664	+	store := newTestStore(t)
	665	+	runtime := outmocks.NewMockContainerRuntime(t)
	666	+	images := outmocks.NewMockImageResolver(t)
	667	+	secrets := outmocks.NewMockSecretProvider(t)
	668	+	seedPendingRetirement(t, ctx, store, "c-old")
	669	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	670	+
	671	+	first := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
	672	+	restarted := first.Add(time.Minute)
	673	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: first}, nil).Twice()
	674	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: restarted}, nil).Once()
	675	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-new").Return("", false, nil)
	676	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	677	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil)
	678	+	require.NoError(t, svc.ReconcileRunning(ctx))
	679	+
	680	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
	681	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store))
	682	+	}
	683	+
	684	+	// TestReconcileRunning_RetiresStackedPredecessors proves several superseded
	685	+	// containers of one service (deploys that each left one behind) are all
	686	+	// retired once the successor is verified.
	687	+	func TestReconcileRunning_RetiresStackedPredecessors(t *testing.T) {
	688	+	ctx := context.Background()
	689	+	store := newTestStore(t)
	690	+	runtime := outmocks.NewMockContainerRuntime(t)
	691	+	images := outmocks.NewMockImageResolver(t)
	692	+	secrets := outmocks.NewMockSecretProvider(t)
	693	+	seedPendingRetirement(t, ctx, store, "c-old-1", "c-old-2")
	694	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	695	+
	696	+	expectVerifiedRunning(runtime, "c-new", 18081)
	697	+	for _, id := range []string{"c-old-1", "c-old-2"} {
	698	+	runtime.EXPECT().StopContainer(mock.Anything, id, domain.AppDefaultStopGrace).Return(nil).Once()
	699	+	runtime.EXPECT().RemoveContainer(mock.Anything, id, false).Return(nil).Once()
	700	+	}
	701	+	require.NoError(t, svc.ReconcileRunning(ctx))
	702	+	assert.Empty(t, markerIDs(t, ctx, store))
	703	+	}
	704	+
	705	+	// releaseFailingState wraps the real store and fails the first backend claim
	706	+	// releases, like a store write error would.
	707	+	type releaseFailingState struct {
	708	+	*appstate.Store
	709	+	failures int
	710	+	}
	711	+
	712	+	func (s *releaseFailingState) ReleaseBackendBinds(ctx context.Context, app, containerID string) error {
	713	+	if s.failures > 0 {
	714	+	s.failures--
	715	+	return assert.AnError
	716	+	}
	717	+	return s.Store.ReleaseBackendBinds(ctx, app, containerID)
	718	+	}
	719	+
	720	+	// TestReconcileRunning_KeepsMarkerUntilBackendClaimsAreReleased proves a
	721	+	// retirement whose claim release failed does not drop the durable marker: the
	722	+	// leak is reported by the pass and retried (retiring a gone container is
	723	+	// idempotent) until the claims are released.
	724	+	func TestReconcileRunning_KeepsMarkerUntilBackendClaimsAreReleased(t *testing.T) {
	725	+	ctx := context.Background()
	726	+	store := newTestStore(t)
	727	+	runtime := outmocks.NewMockContainerRuntime(t)
	728	+	images := outmocks.NewMockImageResolver(t)
	729	+	secrets := outmocks.NewMockSecretProvider(t)
	730	+	seedPendingRetirement(t, ctx, store, "c-old")
	731	+	state := &releaseFailingState{Store: store, failures: 1}
	732	+	traffic := &storeTraffic{store: store}
	733	+	svc := deployment.NewService(deployment.Deps{
	734	+	State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
	735	+	ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
	736	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	737	+	func(context.Context, string, string) (int, error) { return 200, nil },
	738	+	func(context.Context, string) error { return nil },
	739	+	))
	740	+
	741	+	expectVerifiedRunning(runtime, "c-new", 18081)
	742	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Twice()
	743	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	744	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
	745	+
	746	+	err := svc.ReconcileRunning(ctx)
	747	+	require.Error(t, err, "the leaked claims are reported, not discarded")
	748	+	assert.Contains(t, err.Error(), "release backend claims")
	749	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "the marker stays until the claims are released")
	750	+
	751	+	require.NoError(t, svc.ReconcileRunning(ctx), "not-found counts as gone, so the retry settles it")
	752	+	assert.Empty(t, markerIDs(t, ctx, store))
	753	+	}
	754	+
	755	+	// TestStop_RetiresPendingSupersededContainers proves a stopped app does not
	756	+	// keep a superseded predecessor running, and that one which cannot be retired
	757	+	// fails the stop and keeps its marker.
	758	+	func TestStop_RetiresPendingSupersededContainers(t *testing.T) {
	759	+	ctx := context.Background()
	760	+	store := newTestStore(t)
	761	+	runtime := outmocks.NewMockContainerRuntime(t)
	762	+	images := outmocks.NewMockImageResolver(t)
	763	+	secrets := outmocks.NewMockSecretProvider(t)
	764	+	seedPendingRetirement(t, ctx, store, "c-old")
	765	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	766	+
	767	+	runtime.EXPECT().StopContainer(mock.Anything, "c-new", mock.Anything).Return(nil).Twice()
	768	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", false).Return(nil).Twice()
	769	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Twice()
	770	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
	771	+
	772	+	result, err := svc.Stop(ctx, "blog", "")
	773	+	require.Error(t, err, "a predecessor that is still running fails the stop")
	774	+	assert.Equal(t, "failed", result.Services["web"].Result)
	775	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "its marker stays for the next attempt")
	776	+
	777	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	778	+	_, err = svc.Stop(ctx, "blog", "")
	779	+	require.NoError(t, err)
	780	+	assert.Empty(t, markerIDs(t, ctx, store))
	781	+	}
	782	+
	783	+	// TestRemove_FailsAndKeepsStateWhenAPendingPredecessorCannotBeRetired proves
	784	+	// Remove does not end the incarnation (which deletes the markers) while a
	785	+	// superseded container is still running.
	786	+	func TestRemove_FailsAndKeepsStateWhenAPendingPredecessorCannotBeRetired(t *testing.T) {
	787	+	ctx := context.Background()
	788	+	store := newTestStore(t)
	789	+	runtime := outmocks.NewMockContainerRuntime(t)
	790	+	images := outmocks.NewMockImageResolver(t)
	791	+	secrets := outmocks.NewMockSecretProvider(t)
	792	+	seedPendingRetirement(t, ctx, store, "c-old")
	793	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	794	+
	795	+	runtime.EXPECT().StopContainer(mock.Anything, "c-new", mock.Anything).Return(nil).Once()
	796	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", false).Return(nil).Once()
	797	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	798	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
	799	+
	800	+	_, err := svc.Remove(ctx, "blog", "")
	801	+	require.Error(t, err)
	802	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "the marker survives so the container stays tracked")
	803	+	_, ok, loadErr := store.LoadActive(ctx, "blog")
	804	+	require.NoError(t, loadErr)
	805	+	assert.True(t, ok, "the incarnation is not ended")
	806	+	}
	807	+
	808	+	// inhibitionsFailingState wraps the real store and fails the load of the
	809	+	// recovery inhibitions, which is how Stop learns about pending predecessors.
	810	+	type inhibitionsFailingState struct {
	811	+	*appstate.Store
	812	+	}
	813	+
	814	+	func (s *inhibitionsFailingState) LoadRecoveryInhibitions(context.Context, string) ([]domain.AppRecoveryInhibition, error) {
	815	+	return nil, assert.AnError
	816	+	}
	817	+
	818	+	// TestStop_PendingRetirementLoadFailureIsAJournaledFailure proves an app-level
	819	+	// cleanup failure (the pending predecessors cannot even be listed) is not lost
	820	+	// in the per-service results: the operation is journaled as failed, so Stop
	821	+	// errors and a replay of the same key reads the same failure instead of a
	822	+	// success.
	823	+	func TestStop_PendingRetirementLoadFailureIsAJournaledFailure(t *testing.T) {
	824	+	ctx := context.Background()
	825	+	store := newTestStore(t)
	826	+	runtime := outmocks.NewMockContainerRuntime(t)
	827	+	seedPendingRetirement(t, ctx, store, "c-old")
	828	+	// The ACTIVE container is retired successfully before the load fails.
	829	+	runtime.EXPECT().StopContainer(mock.Anything, "c-new", mock.Anything).Return(nil).Once()
	830	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", false).Return(nil).Once()
	831	+	svc := deployment.NewService(deployment.Deps{
	832	+	State: &inhibitionsFailingState{Store: store}, Runtime: runtime, Traffic: &storeTraffic{store: store},
	833	+	}, zerowrap.Default())
	834	+
	835	+	result, err := svc.Stop(ctx, "blog", "key-stop")
	836	+	require.Error(t, err)
	837	+	assert.Equal(t, "deployed", result.Services["web"].Result, "the ACTIVE container really was stopped")
	838	+	require.NotEmpty(t, result.Warnings, "the app-level failure is surfaced on the result")
	839	+
	840	+	op, loadErr := store.LoadOperation(ctx, "blog", "key-stop")
	841	+	require.NoError(t, loadErr)
	842	+	assert.Equal(t, domain.AppOutcomeFailed, op.Outcome, "the journal records the failure, not the per-service success")
	843	+	step, found := opStep(op, "pending.retire")
	844	+	require.True(t, found)
	845	+	assert.Equal(t, domain.AppStepFailed, step.State)
	846	+	assert.NotEmpty(t, step.Error)
	847	+	require.NotEmpty(t, op.Warnings)
	848	+	assert.Contains(t, op.Warnings[0].Detail, "pending retirements unknown")
	849	+
	850	+	// Replaying the same key returns the same failure and runs nothing again.
	851	+	_, replayErr := svc.Stop(ctx, "blog", "key-stop")
	852	+	require.Error(t, replayErr)
	853	+	assert.ErrorIs(t, replayErr, domain.ErrAppStateConflict)
	854	+	assert.Contains(t, replayErr.Error(), domain.AppOutcomeFailed)
	855	+	runtime.AssertNumberOfCalls(t, "StopContainer", 1)
	856	+	}
	857	+
	858	+	// executionStartedAt is the start time the fakes report for a freshly started
	859	+	// container: deploy and restart observe it right after the start.
	860	+	var executionStartedAt = time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
	861	+
	862	+	// dropInspectExpectations removes the InspectContainer expectations so a test
	863	+	// can report a different execution after a deploy.
	864	+	func dropInspectExpectations(runtime *outmocks.MockContainerRuntime) {
	865	+	kept := runtime.ExpectedCalls[:0]
	866	+	for _, call := range runtime.ExpectedCalls {
	867	+	if call.Method != "InspectContainer" {
	868	+	kept = append(kept, call)
	869	+	}
	870	+	}
	871	+	runtime.ExpectedCalls = kept
	872	+	}
	873	+
	874	+	// TestReconcileRunning_DoesNotWithdrawAFreshlyDeployedGeneration is the
	875	+	// regression for a deployed generation being withdrawn by the first periodic
	876	+	// pass: the deploy verified and routed that execution, so the pass must treat
	877	+	// it as already seen. The traffic fake mutates ACTIVE like the real publisher,
	878	+	// so a withdrawal would drop the service's routed binds.
	879	+	func TestReconcileRunning_DoesNotWithdrawAFreshlyDeployedGeneration(t *testing.T) {
	880	+	ctx := context.Background()
	881	+	store := newTestStore(t)
	882	+	runtime := outmocks.NewMockContainerRuntime(t)
	883	+	images := outmocks.NewMockImageResolver(t)
	884	+	secrets := outmocks.NewMockSecretProvider(t)
	885	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	886	+	traffic := &storeTraffic{store: store}
	887	+	svc := overlapService(store, runtime, images, secrets, traffic)
	888	+
	889	+	overlapDeployMocks(runtime, images)
	890	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	891	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	892	+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	893	+	require.NoError(t, err)
	894	+	require.Equal(t, map[int]int{8080: 18081}, traffic.lastRoute("web"))
	895	+
	896	+	// The first periodic pass sees the same execution the deploy verified.
	897	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-new").Return("", false, nil)
	898	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	899	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil)
	900	+	require.NoError(t, svc.ReconcileRunning(ctx))
	901	+
	902	+	assert.Empty(t, traffic.withdrawals(), "the verified generation is never withdrawn")
	903	+	for i, routed := range traffic.allRoutes("web") {
	904	+	assert.Equal(t, map[int]int{8080: 18081}, routed, "graph %d kept routing the service", i)
	905	+	}
	906	+	active, _, err := store.LoadActive(ctx, "blog")
	907	+	require.NoError(t, err)
	908	+	assert.Equal(t, map[int]int{8080: 18081}, active.Services["web"].BackendBinds)
	909	+	}
	910	+
	911	+	// TestReconcileRunning_TreatsARestartedExecutionAfterDeployAsFresh proves the
	912	+	// recorded execution is the one observed before readiness: if the container
	913	+	// restarted natively afterwards, its start time differs and the pass withdraws
	914	+	// and verifies it again.
	915	+	func TestReconcileRunning_TreatsARestartedExecutionAfterDeployAsFresh(t *testing.T) {
	916	+	ctx := context.Background()
	917	+	store := newTestStore(t)
	918	+	runtime := outmocks.NewMockContainerRuntime(t)
	919	+	images := outmocks.NewMockImageResolver(t)
	920	+	secrets := outmocks.NewMockSecretProvider(t)
	921	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	922	+	traffic := &storeTraffic{store: store}
	923	+	svc := overlapService(store, runtime, images, secrets, traffic)
	924	+
	925	+	overlapDeployMocks(runtime, images)
	926	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	927	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	928	+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	929	+	require.NoError(t, err)
	930	+
	931	+	dropInspectExpectations(runtime)
	932	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(
	933	+	&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt.Add(time.Minute)}, nil)
	934	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-new").Return("", false, nil)
	935	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	936	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18090, Protocol: domain.NetworkProtocolTCP}}, nil)
	937	+	require.NoError(t, svc.ReconcileRunning(ctx))
	938	+
	939	+	assert.Equal(t, []string{"blog/web"}, traffic.withdrawals(), "a different execution is withdrawn before it is verified")
	940	+	assert.Equal(t, map[int]int{8080: 18090}, traffic.lastRoute("web"), "and routed again on its verified binds")
	941	+	}
	942	+
	943	+	// TestReconcileRunning_DoesNotWithdrawARestartedGeneration covers the restart
	944	+	// path: an in-place restart verified and routed the new execution, so the
	945	+	// first periodic pass keeps it.
	946	+	func TestReconcileRunning_DoesNotWithdrawARestartedGeneration(t *testing.T) {
	947	+	ctx := context.Background()
	948	+	store := newTestStore(t)
	949	+	runtime := outmocks.NewMockContainerRuntime(t)
	950	+	images := outmocks.NewMockImageResolver(t)
	951	+	secrets := outmocks.NewMockSecretProvider(t)
	952	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	953	+	traffic := &storeTraffic{store: store}
	954	+	svc := overlapService(store, runtime, images, secrets, traffic)
	955	+
	956	+	restartedAt := executionStartedAt.Add(time.Hour)
	957	+	runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	958	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(
	959	+	&domain.Container{ID: "c-old", Status: "running", StartedAt: restartedAt}, nil)
	960	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
	961	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 17002, Protocol: domain.NetworkProtocolTCP}}, nil)
	962	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-old").Return("", false, nil)
	963	+	_, err := svc.Restart(ctx, "blog", "web", "")
	964	+	require.NoError(t, err)
	965	+	withdrawnByRestart := len(traffic.withdrawals())
	966	+
	967	+	require.NoError(t, svc.ReconcileRunning(ctx))
	968	+	assert.Len(t, traffic.withdrawals(), withdrawnByRestart, "the periodic pass withdraws nothing more")
	969	+	assert.Equal(t, map[int]int{8080: 17002}, traffic.lastRoute("web"))
	970	+	}
	971	+
	972	+	// TestReconcileRunning_RestoresARebuiltGenerationWhoseTrafficWasRejected is the
	973	+	// regression for a missing-container rebuild whose graph apply was rejected:
	974	+	// the rebuilt execution must not be recorded as verified, or the next periodic
	975	+	// pass sees an already verified generation whose binds match ACTIVE, skips the
	976	+	// publication, and leaves the service unrouted indefinitely.
	977	+	func TestReconcileRunning_RestoresARebuiltGenerationWhoseTrafficWasRejected(t *testing.T) {
	978	+	ctx := context.Background()
	979	+	store := newTestStore(t)
	980	+	runtime := outmocks.NewMockContainerRuntime(t)
	981	+	images := outmocks.NewMockImageResolver(t)
	982	+	secrets := outmocks.NewMockSecretProvider(t)
	983	+	seedReplaceableApp(t, ctx, store)
	984	+	traffic := &storeTraffic{store: store, failRebuild: 1}
	985	+	svc := deployment.NewService(deployment.Deps{
	986	+	State: store, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
	987	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	988	+	func(context.Context, string, string) (int, error) { return 200, nil },
	989	+	func(context.Context, string) error { return nil },
	990	+	))
	991	+
	992	+	// The recorded container is gone: restart rebuilds it from the pinned
	993	+	// ACTIVE digest, and the graph apply that follows is rejected.
	994	+	runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
	995	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
	996	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
	997	+	expectNetworkProvision(runtime, "app-blog", 1)
	998	+	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
	999	+	runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
	1000	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuilt").Return(
	1001	+	&domain.Container{ID: "c-rebuilt", Status: "running", StartedAt: executionStartedAt}, nil)
	1002	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
	1003	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil)
	1004	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-rebuilt").Return("", false, nil)
	1005	+
	1006	+	_, err := svc.Restart(ctx, "blog", "web", "op-restart")
	1007	+	require.Error(t, err, "a rejected graph is a failed restart")
	1008	+	require.Zero(t, traffic.rebuildCount(), "no graph was accepted")
	1009	+	active, _, loadErr := store.LoadActive(ctx, "blog")
	1010	+	require.NoError(t, loadErr)
	1011	+	require.Equal(t, "c-rebuilt", active.Services["web"].Container)
	1012	+
	1013	+	// The next periodic pass must verify and route the generation itself.
	1014	+	require.NoError(t, svc.ReconcileRunning(ctx))
	1015	+	assert.Equal(t, map[int]int{8080: 18082}, traffic.lastRoute("web"), "the rebuilt generation is routed again")
	1016	+	}

@@ -0,0 +1,1016 @@
	1	+	package deployment_test
	2	+
	3	+	import (
	4	+	"context"
	5	+	"maps"
	6	+	"os"
	7	+	"path/filepath"
	8	+	"sync"
	9	+	"testing"
	10	+	"time"
	11	+
	12	+	"github.com/bnema/zerowrap"
	13	+	"github.com/stretchr/testify/assert"
	14	+	"github.com/stretchr/testify/mock"
	15	+	"github.com/stretchr/testify/require"
	16	+
	17	+	"github.com/bnema/gordon/internal/adapters/out/appstate"
	18	+	outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
	19	+	"github.com/bnema/gordon/internal/domain"
	20	+	"github.com/bnema/gordon/internal/usecase/deployment"
	21	+	)
	22	+
	23	+	// TestDeploy_NonOverlapEligibleServicesKeepSequentialOrder proves a service
	24	+	// with a volume, a bind, or a device is never run beside its predecessor: the
	25	+	// service is withdrawn, the superseded container is stopped and removed, and
	26	+	// only then is the replacement created.
	27	+	func TestDeploy_NonOverlapEligibleServicesKeepSequentialOrder(t *testing.T) {
	28	+	root := t.TempDir()
	29	+	source := filepath.Join(root, "binds", "config")
	30	+	require.NoError(t, os.MkdirAll(source, 0o755))
	31	+
	32	+	cases := []struct {
	33	+	name string
	34	+	mutate func(*domain.AppService)
	35	+	inhibits bool
	36	+	cdiSupported bool
	37	+	}{
	38	+	{"volume", func(s *domain.AppService) { s.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}} }, true, false},
	39	+	{"bind", func(s *domain.AppService) {
	40	+	s.Binds = []domain.AppBind{{Name: "config", Path: "/etc/app.conf", ReadOnly: true}}
	41	+	}, true, false},
	42	+	{"device", func(s *domain.AppService) { s.Devices = []string{"test_gpu"} }, false, true},
	43	+	{"udp", func(s *domain.AppService) {
	44	+	s.UDP = []domain.AppUDPInterface{{Entrypoint: "game", Port: 28015, Publish: "0.0.0.0:28015"}}
	45	+	}, false, false},
	46	+	}
	47	+	for _, tc := range cases {
	48	+	t.Run(tc.name, func(t *testing.T) {
	49	+	ctx := context.Background()
	50	+	state, runtime, images, secrets := mockDeps(t)
	51	+	rev := mockRevision()
	52	+	tc.mutate(&rev.Spec.Services[0])
	53	+
	54	+	expectDeployPreflight(state, runtime, images, secrets, rev)
	55	+	runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil).Maybe()
	56	+	runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe()
	57	+	if tc.cdiSupported {
	58	+	runtime.EXPECT().SupportsCDIDevices(mock.Anything).Return(nil)
	59	+	}
	60	+	expectNetworkProvision(runtime, "app-blog", 1)
	61	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
	62	+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	63	+	if tc.inhibits {
	64	+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
	65	+	}
	66	+
	67	+	var order []string
	68	+	expectRetireOld(state, runtime, &order, "c-old")
	69	+	expectSequentialCreate(state, runtime, &order, "c-new", 18080)
	70	+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(
	71	+	func(context.Context, domain.AppActive) error {
	72	+	recordEvent(&order, "publish-active")
	73	+	return nil
	74	+	}).Once()
	75	+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
	76	+	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
	77	+
	78	+	svc := deployment.NewService(deployment.Deps{
	79	+	State: state, Runtime: runtime, Images: images, Secrets: secrets,
	80	+	Traffic: &orderedTraffic{events: &order},
	81	+	}, zerowrap.Default()).
	82	+	WithBindPolicies(map[string]domain.AppBindPolicy{"config": {
	83	+	Name: "config", Source: source, Root: root, AllowedApps: []string{"blog"}, AllowedServices: []string{"web"},
	84	+	}}).
	85	+	WithDevicePolicies(map[string]domain.AppDevicePolicy{"test_gpu": {
	86	+	Name: "test_gpu", CDI: []string{"example.com/gpu=GPU-test-uuid"},
	87	+	AllowedApps: []string{"blog"}, AllowedServices: []string{"web"},
	88	+	}}).
	89	+	WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	90	+	func(context.Context, string, string) (int, error) { return 200, nil },
	91	+	func(context.Context, string) error { return nil },
	92	+	))
	93	+
	94	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	95	+	require.NoError(t, err)
	96	+	assert.Equal(t, "deployed", result.Services["web"].Result)
	97	+	assert.Equal(t,
	98	+	[]string{"withdraw", "stop-old", "remove-old", "create-new", "start-new", "publish-active", "traffic"},
	99	+	order, "the superseded container is gone before the replacement is created")
	100	+	})
	101	+	}
	102	+	}
	103	+
	104	+	// TestDeploy_OverlapPublicationFailureKeepsOldContainer proves a failed ACTIVE
	105	+	// publication of an overlap replacement never stops the superseded container,
	106	+	// which still serves because ACTIVE and routing were not switched. The
	107	+	// candidate stays journaled as the step's After, so reconciliation removes it,
	108	+	// and the failure is reported with a leftover warning on the old container.
	109	+	func TestDeploy_OverlapPublicationFailureKeepsOldContainer(t *testing.T) {
	110	+	ctx := context.Background()
	111	+	state, runtime, images, secrets := mockDeps(t)
	112	+	rev := mockRevision()
	113	+
	114	+	expectDeployPreflight(state, runtime, images, secrets, rev)
	115	+	expectNetworkProvision(runtime, "app-blog", 1)
	116	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
	117	+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	118	+	expectSequentialCreate(state, runtime, nil, "c-new", 18080)
	119	+	expectRetirementMarker(state, "c-old")
	120	+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(assert.AnError).Once()
	121	+
	122	+	var events []string
	123	+	svc := deployment.NewService(deployment.Deps{
	124	+	State: state, Runtime: runtime, Images: images, Secrets: secrets,
	125	+	Traffic: &orderedTraffic{events: &events},
	126	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	127	+	func(context.Context, string, string) (int, error) { return 200, nil },
	128	+	func(context.Context, string) error { return nil },
	129	+	))
	130	+
	131	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	132	+	require.Error(t, err)
	133	+	assert.Equal(t, "failed", result.Services["web"].Result)
	134	+	assert.Equal(t, "c-new", result.Services["web"].After)
	135	+	require.Len(t, result.Services["web"].CleanupWarnings, 1)
	136	+	assert.Equal(t, "c-old", result.Services["web"].CleanupWarnings[0].Leftover)
	137	+	assert.Empty(t, events, "the proxy graph was neither withdrawn nor rebuilt")
	138	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
	139	+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
	140	+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
	141	+	}
	142	+
	143	+	// storeTraffic is a traffic publisher that behaves like the real one against
	144	+	// a real app-state store: a withdrawal drops the service's recorded binds from
	145	+	// ACTIVE (and can then report a failed graph apply), and a rebuild routes
	146	+	// exactly the binds ACTIVE records at that moment. Tests assert on what is
	147	+	// routed, not on call order.
	148	+	type storeTraffic struct {
	149	+	mu sync.Mutex
	150	+	store *appstate.Store
	151	+	failWithdraw int
	152	+	failRebuild int
	153	+	withdrawn []string
	154	+	routed []map[string]map[int]int
	155	+	}
	156	+
	157	+	func (t *storeTraffic) WithdrawService(ctx context.Context, app, service string) error {
	158	+	t.mu.Lock()
	159	+	defer t.mu.Unlock()
	160	+	t.withdrawn = append(t.withdrawn, app+"/"+service)
	161	+	active, ok, err := t.store.LoadActive(ctx, app)
	162	+	if err != nil \|\| !ok {
	163	+	return err
	164	+	}
	165	+	if eff, found := active.Services[service]; found {
	166	+	eff.BackendBinds, eff.UDPBackendBinds = nil, nil
	167	+	active.Services[service] = eff
	168	+	if err := t.store.SaveActive(ctx, active); err != nil {
	169	+	return err
	170	+	}
	171	+	}
	172	+	if t.failWithdraw > 0 {
	173	+	t.failWithdraw--
	174	+	return assert.AnError
	175	+	}
	176	+	return nil
	177	+	}
	178	+
	179	+	func (t *storeTraffic) WithdrawServiceState(ctx context.Context, app, service string) error {
	180	+	return t.WithdrawService(ctx, app, service)
	181	+	}
	182	+
	183	+	func (t *storeTraffic) RebuildTraffic(ctx context.Context) error {
	184	+	t.mu.Lock()
	185	+	defer t.mu.Unlock()
	186	+	if t.failRebuild > 0 {
	187	+	t.failRebuild--
	188	+	return assert.AnError
	189	+	}
	190	+	active, _, err := t.store.LoadActive(ctx, "blog")
	191	+	if err != nil {
	192	+	return err
	193	+	}
	194	+	routes := map[string]map[int]int{}
	195	+	for name, eff := range active.Services {
	196	+	routes[name] = maps.Clone(eff.BackendBinds)
	197	+	}
	198	+	t.routed = append(t.routed, routes)
	199	+	return nil
	200	+	}
	201	+
	202	+	// lastRoute returns the binds the most recent graph routed for a service.
	203	+	func (t *storeTraffic) lastRoute(service string) map[int]int {
	204	+	t.mu.Lock()
	205	+	defer t.mu.Unlock()
	206	+	if len(t.routed) == 0 {
	207	+	return nil
	208	+	}
	209	+	return t.routed[len(t.routed)-1][service]
	210	+	}
	211	+
	212	+	// allRoutes returns the binds of every graph routed for a service.
	213	+	func (t *storeTraffic) allRoutes(service string) []map[int]int {
	214	+	t.mu.Lock()
	215	+	defer t.mu.Unlock()
	216	+	var routes []map[int]int
	217	+	for _, routed := range t.routed {
	218	+	routes = append(routes, routed[service])
	219	+	}
	220	+	return routes
	221	+	}
	222	+
	223	+	// withdrawals returns the services withdrawn so far, in order.
	224	+	func (t *storeTraffic) withdrawals() []string {
	225	+	t.mu.Lock()
	226	+	defer t.mu.Unlock()
	227	+	return append([]string(nil), t.withdrawn...)
	228	+	}
	229	+
	230	+	func (t *storeTraffic) rebuildCount() int {
	231	+	t.mu.Lock()
	232	+	defer t.mu.Unlock()
	233	+	return len(t.routed)
	234	+	}
	235	+
	236	+	const overlapImage = "registry.example.com/blog/web:1.4.2"
	237	+
	238	+	// overlapWebSpec is a stateless public HTTP service.
	239	+	func overlapWebSpec(grace time.Duration) domain.AppService {
	240	+	return domain.AppService{
	241	+	Name: "web", Image: overlapImage, Secrets: map[string]string{}, StopGrace: grace,
	242	+	HTTP: []domain.AppHTTPInterface{{Host: "blog.example.com", Port: 8080, TLS: domain.AppTLSAuto}},
	243	+	Readiness: domain.AppReadiness{
	244	+	Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: time.Second,
	245	+	},
	246	+	}
	247	+	}
	248	+
	249	+	func overlapOldBinds() map[int]int { return map[int]int{8080: 17000} }
	250	+
	251	+	// seedServingApp seeds an app whose container c-old serves on 17000, with a
	252	+	// materialized revision to deploy over it.
	253	+	func seedServingApp(t testing.T, ctx context.Context, store appstate.Store, spec domain.AppService) {
	254	+	t.Helper()
	255	+	require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
	256	+	require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
	257	+	require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	258	+	"web": {Container: "c-old", EffectiveRevision: "rev-0", Spec: spec, BackendBinds: overlapOldBinds()},
	259	+	}}))
	260	+	seedDesired(t, ctx, store, testRevision("blog", spec))
	261	+	}
	262	+
	263	+	// overlapDeployDeps wires the preflight and candidate creation of one overlap
	264	+	// deploy that creates c-new on 18081 and expects readiness to pass.
	265	+	func overlapDeployMocks(runtime outmocks.MockContainerRuntime, images outmocks.MockImageResolver) {
	266	+	images.EXPECT().ResolveDigest(mock.Anything, overlapImage).Return(restartTestDigest, nil).Once()
	267	+	runtime.EXPECT().InspectImageVolumes(mock.Anything, overlapImage).Return(nil, nil).Once()
	268	+	expectNetworkProvision(runtime, "app-blog", 1)
	269	+	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
	270	+	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	271	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
	272	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	273	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
	274	+	}
	275	+
	276	+	func overlapService(store appstate.Store, runtime outmocks.MockContainerRuntime, images outmocks.MockImageResolver, secrets outmocks.MockSecretProvider, traffic storeTraffic) deployment.Service {
	277	+	return deployment.NewService(deployment.Deps{
	278	+	State: store, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
	279	+	ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
	280	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	281	+	func(context.Context, string, string) (int, error) { return 200, nil },
	282	+	func(context.Context, string) error { return nil },
	283	+	))
	284	+	}
	285	+
	286	+	// expectVerifiedRunning wires one recovery verification of a running
	287	+	// container that publishes the given loopback port.
	288	+	func expectVerifiedRunning(runtime *outmocks.MockContainerRuntime, id string, port int) {
	289	+	started := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
	290	+	runtime.EXPECT().InspectContainer(mock.Anything, id).
	291	+	Return(&domain.Container{ID: id, Status: "running", StartedAt: started}, nil)
	292	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, id).Return("", false, nil)
	293	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, id, mock.Anything).Return(
	294	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: port, Protocol: domain.NetworkProtocolTCP}}, nil)
	295	+	}
	296	+
	297	+	// TestDeploy_OverlapSwitchIsNotErasedByAPendingWithdrawal proves that a
	298	+	// publication withdrawal left pending by an earlier failed recovery does not
	299	+	// unroute the new generation: the deploy resolves it before publishing, so
	300	+	// the graph the switch applies routes the candidate's binds. The traffic fake
	301	+	// mutates ACTIVE like the real publisher, so a stray withdrawal after the
	302	+	// switch would leave the service unrouted.
	303	+	func TestDeploy_OverlapSwitchIsNotErasedByAPendingWithdrawal(t *testing.T) {
	304	+	ctx := context.Background()
	305	+	store := newTestStore(t)
	306	+	runtime := outmocks.NewMockContainerRuntime(t)
	307	+	images := outmocks.NewMockImageResolver(t)
	308	+	secrets := outmocks.NewMockSecretProvider(t)
	309	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	310	+	traffic := &storeTraffic{store: store, failWithdraw: 1}
	311	+	svc := overlapService(store, runtime, images, secrets, traffic)
	312	+
	313	+	// A recovery pass finds c-old down and cannot prove its withdrawal: the
	314	+	// service is marked as having a pending withdrawal.
	315	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").
	316	+	Return(&domain.Container{ID: "c-old", Status: "exited"}, nil).Once()
	317	+	require.Error(t, svc.ReconcileRunning(ctx))
	318	+
	319	+	overlapDeployMocks(runtime, images)
	320	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", domain.AppDefaultStopGrace).Return(nil).Once()
	321	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	322	+
	323	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	324	+	require.NoError(t, err)
	325	+	assert.Equal(t, "deployed", result.Services["web"].Result)
	326	+
	327	+	assert.Equal(t, map[int]int{8080: 18081}, traffic.lastRoute("web"), "the service routes the new generation")
	328	+	active, _, err := store.LoadActive(ctx, "blog")
	329	+	require.NoError(t, err)
	330	+	assert.Equal(t, "c-new", active.Services["web"].Container)
	331	+	assert.Equal(t, map[int]int{8080: 18081}, active.Services["web"].BackendBinds,
	332	+	"ACTIVE keeps only the new generation's binds")
	333	+	}
	334	+
	335	+	// TestDeploy_OverlapRetirementSurvivesANewerOperation proves the cleanup of a
	336	+	// superseded container is owned by the durable retirement marker, not by the
	337	+	// operation journal: after the first retirement fails and a newer operation
	338	+	// becomes the latest journal, the periodic reconciliation still retires it,
	339	+	// with the predecessor's own stop grace, and only once its successor is
	340	+	// verified and routed.
	341	+	func TestDeploy_OverlapRetirementSurvivesANewerOperation(t *testing.T) {
	342	+	ctx := context.Background()
	343	+	store := newTestStore(t)
	344	+	runtime := outmocks.NewMockContainerRuntime(t)
	345	+	images := outmocks.NewMockImageResolver(t)
	346	+	secrets := outmocks.NewMockSecretProvider(t)
	347	+	seedServingApp(t, ctx, store, overlapWebSpec(45*time.Second))
	348	+	traffic := &storeTraffic{store: store}
	349	+	svc := overlapService(store, runtime, images, secrets, traffic)
	350	+
	351	+	overlapDeployMocks(runtime, images)
	352	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", 45*time.Second).Return(nil).Twice()
	353	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
	354	+
	355	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	356	+	require.NoError(t, err, "a retirement leftover does not fail a deploy whose new generation serves")
	357	+	require.Len(t, result.CleanupWarnings, 1)
	358	+	assert.Equal(t, "c-old", result.CleanupWarnings[0].Leftover)
	359	+
	360	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	361	+	require.NoError(t, err)
	362	+	require.Len(t, markers, 1, "the failed retirement leaves its durable marker")
	363	+	assert.Equal(t, "c-old", markers[0].ContainerID)
	364	+	assert.Equal(t, domain.AppInhibitRetirementPending, markers[0].Reason)
	365	+	assert.Equal(t, 45*time.Second, markers[0].StopGrace, "the predecessor's grace travels with the marker")
	366	+
	367	+	// A newer operation becomes the latest journal and masks the deploy's.
	368	+	require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
	369	+	Op: "op-newer", Kind: "deploy", App: "blog", StartedAt: time.Now().Add(time.Minute).UTC(),
	370	+	Outcome: domain.AppOutcomeSuccess,
	371	+	}))
	372	+
	373	+	expectVerifiedRunning(runtime, "c-new", 18082)
	374	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	375	+	require.NoError(t, svc.ReconcileRunning(ctx))
	376	+
	377	+	markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
	378	+	require.NoError(t, err)
	379	+	assert.Empty(t, markers, "the marker is cleared once the container is confirmed gone")
	380	+	assert.Equal(t, map[int]int{8080: 18082}, traffic.lastRoute("web"), "the verified successor is routed")
	381	+	active, _, err := store.LoadActive(ctx, "blog")
	382	+	require.NoError(t, err)
	383	+	assert.Equal(t, "c-new", active.Services["web"].Container)
	384	+	}
	385	+
	386	+	// TestReconcileRunning_RetiresSupersededOnlyAfterSuccessorIsVerified proves a
	387	+	// superseded container is kept while its successor cannot be verified: the
	388	+	// working generation is never stopped in favour of one that is not serving.
	389	+	func TestReconcileRunning_RetiresSupersededOnlyAfterSuccessorIsVerified(t *testing.T) {
	390	+	ctx := context.Background()
	391	+	store := newTestStore(t)
	392	+	runtime := outmocks.NewMockContainerRuntime(t)
	393	+	images := outmocks.NewMockImageResolver(t)
	394	+	secrets := outmocks.NewMockSecretProvider(t)
	395	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	396	+	require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	397	+	"web": {Container: "c-new", EffectiveRevision: "rev-1", Spec: overlapWebSpec(0), BackendBinds: map[int]int{8080: 18081}},
	398	+	}}))
	399	+	require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
	400	+	App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitRetirementPending,
	401	+	}))
	402	+	traffic := &storeTraffic{store: store}
	403	+	svc := overlapService(store, runtime, images, secrets, traffic)
	404	+
	405	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(nil, assert.AnError).Once()
	406	+	require.Error(t, svc.ReconcileRunning(ctx))
	407	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
	408	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	409	+	require.NoError(t, err)
	410	+	assert.Len(t, markers, 1, "the marker waits for a verified successor")
	411	+
	412	+	// A record written before the grace was persisted falls back to the default.
	413	+	expectVerifiedRunning(runtime, "c-new", 18081)
	414	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", domain.AppDefaultStopGrace).Return(nil).Once()
	415	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	416	+	require.NoError(t, svc.ReconcileRunning(ctx))
	417	+	markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
	418	+	require.NoError(t, err)
	419	+	assert.Empty(t, markers)
	420	+	}
	421	+
	422	+	// TestReconcileRunning_NeverRetiresAPendingContainerThatActiveNamesAgain
	423	+	// proves a rollback to the superseded container is safe: when ACTIVE names the
	424	+	// container again, its stale retirement marker is dropped and it is kept.
	425	+	func TestReconcileRunning_NeverRetiresAPendingContainerThatActiveNamesAgain(t *testing.T) {
	426	+	ctx := context.Background()
	427	+	store := newTestStore(t)
	428	+	runtime := outmocks.NewMockContainerRuntime(t)
	429	+	images := outmocks.NewMockImageResolver(t)
	430	+	secrets := outmocks.NewMockSecretProvider(t)
	431	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	432	+	require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
	433	+	App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitRetirementPending,
	434	+	}))
	435	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	436	+
	437	+	expectVerifiedRunning(runtime, "c-old", 17000)
	438	+	require.NoError(t, svc.ReconcileRunning(ctx))
	439	+
	440	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
	441	+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
	442	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	443	+	require.NoError(t, err)
	444	+	assert.Empty(t, markers)
	445	+	active, _, err := store.LoadActive(ctx, "blog")
	446	+	require.NoError(t, err)
	447	+	assert.Equal(t, "c-old", active.Services["web"].Container)
	448	+	}
	449	+
	450	+	// TestBootRecovery_DoesNotRepublishUnverifiedBindsOfAPublishedCandidate proves
	451	+	// interrupted-operation reconciliation never rebuilds traffic from the binds
	452	+	// ACTIVE persisted for a published candidate: a restarted candidate's
	453	+	// ephemeral port may belong to another workload. Only the verified start path
	454	+	// routes, and the superseded container is retired afterwards through its
	455	+	// marker.
	456	+	func TestBootRecovery_DoesNotRepublishUnverifiedBindsOfAPublishedCandidate(t *testing.T) {
	457	+	ctx := context.Background()
	458	+	store := newTestStore(t)
	459	+	runtime := outmocks.NewMockContainerRuntime(t)
	460	+	images := outmocks.NewMockImageResolver(t)
	461	+	secrets := outmocks.NewMockSecretProvider(t)
	462	+	spec := overlapWebSpec(0)
	463	+	require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
	464	+	require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	465	+	"web": {Container: "c-new", EffectiveRevision: "rev-1", Spec: spec, BackendBinds: map[int]int{8080: 18081}},
	466	+	}}))
	467	+	require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
	468	+	Op: "op-switch", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
	469	+	Steps: []domain.AppOperationStep{
	470	+	{ID: "preflight", State: domain.AppStepSucceeded},
	471	+	{ID: "service.web.replace", State: domain.AppStepPending, Service: "web", Before: "c-old", After: "c-new"},
	472	+	},
	473	+	}))
	474	+	require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
	475	+	App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitRetirementPending,
	476	+	}))
	477	+	traffic := &storeTraffic{store: store}
	478	+	svc := overlapService(store, runtime, images, secrets, traffic)
	479	+
	480	+	// The candidate restarted and now publishes a different port.
	481	+	runtime.EXPECT().IsContainerRunning(mock.Anything, "c-new").Return(true, nil).Once()
	482	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
	483	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	484	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18090, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
	485	+	require.NoError(t, svc.ReconcileBoot(ctx))
	486	+
	487	+	require.Positive(t, traffic.rebuildCount())
	488	+	for _, routed := range traffic.allRoutes("web") {
	489	+	assert.NotEqual(t, map[int]int{8080: 18081}, routed, "the unverified persisted port is never routed")
	490	+	}
	491	+	assert.Equal(t, map[int]int{8080: 18090}, traffic.lastRoute("web"), "the verified bind is routed")
	492	+	active, _, err := store.LoadActive(ctx, "blog")
	493	+	require.NoError(t, err)
	494	+	assert.Equal(t, map[int]int{8080: 18090}, active.Services["web"].BackendBinds)
	495	+
	496	+	// Boot keeps the superseded container and its marker: the periodic pass
	497	+	// retires it once the successor is verified.
	498	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
	499	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	500	+	require.NoError(t, err)
	501	+	require.Len(t, markers, 1)
	502	+	assert.Equal(t, "c-old", markers[0].ContainerID)
	503	+	}
	504	+
	505	+	// TestDeploy_OverlapTrafficFailureIsRecoveredByThePeriodicPass proves the
	506	+	// failed switch is recoverable and traceable: the graph apply is rejected after
	507	+	// ACTIVE named the new container, the superseded container keeps running with
	508	+	// its marker, the next deploy-side reconciliation keeps that marker, and the
	509	+	// periodic pass retires the old container once the successor is routed.
	510	+	func TestDeploy_OverlapTrafficFailureIsRecoveredByThePeriodicPass(t *testing.T) {
	511	+	ctx := context.Background()
	512	+	store := newTestStore(t)
	513	+	runtime := outmocks.NewMockContainerRuntime(t)
	514	+	images := outmocks.NewMockImageResolver(t)
	515	+	secrets := outmocks.NewMockSecretProvider(t)
	516	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	517	+	traffic := &storeTraffic{store: store, failRebuild: 1}
	518	+	svc := overlapService(store, runtime, images, secrets, traffic)
	519	+
	520	+	overlapDeployMocks(runtime, images)
	521	+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	522	+	require.Error(t, err, "a rejected graph is a failed deploy")
	523	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
	524	+
	525	+	active, _, err := store.LoadActive(ctx, "blog")
	526	+	require.NoError(t, err)
	527	+	assert.Equal(t, "c-new", active.Services["web"].Container, "ACTIVE names the new container")
	528	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	529	+	require.NoError(t, err)
	530	+	require.Len(t, markers, 1)
	531	+	assert.Equal(t, "c-old", markers[0].ContainerID)
	532	+
	533	+	// The next mutation reconciles the failed deploy: the published candidate
	534	+	// is kept, and so is the retirement marker.
	535	+	require.NoError(t, svc.AbandonDeploy(ctx, deployment.DeployClaim{App: "blog", Op: "op-none"}))
	536	+	markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
	537	+	require.NoError(t, err)
	538	+	require.Len(t, markers, 1, "a published candidate's reconciliation keeps the retirement marker")
	539	+
	540	+	expectVerifiedRunning(runtime, "c-new", 18081)
	541	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", domain.AppDefaultStopGrace).Return(nil).Once()
	542	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	543	+	require.NoError(t, svc.ReconcileRunning(ctx))
	544	+	assert.Equal(t, map[int]int{8080: 18081}, traffic.lastRoute("web"))
	545	+	markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
	546	+	require.NoError(t, err)
	547	+	assert.Empty(t, markers)
	548	+	}
	549	+
	550	+	// TestDeploy_PredecessorWithOwnedResourcesNeverOverlaps proves eligibility
	551	+	// looks at the predecessor's effective spec too: a replacement that drops the
	552	+	// UDP interface or the volume of the generation it supersedes still replaces
	553	+	// it sequentially, because the old generation's UDP sessions or writes
	554	+	// outlive the switch.
	555	+	func TestDeploy_PredecessorWithOwnedResourcesNeverOverlaps(t *testing.T) {
	556	+	cases := map[string]func(*domain.AppService){
	557	+	"udp": func(s *domain.AppService) {
	558	+	s.UDP = []domain.AppUDPInterface{{Entrypoint: "game", Port: 28015, Publish: "0.0.0.0:28015"}}
	559	+	},
	560	+	"volume": func(s *domain.AppService) { s.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}} },
	561	+	}
	562	+	for name, mutate := range cases {
	563	+	t.Run(name, func(t *testing.T) {
	564	+	ctx := context.Background()
	565	+	state, runtime, images, secrets := mockDeps(t)
	566	+	rev := mockRevision() // HTTP-only, no volume: eligible on its own
	567	+	previous := rev.Spec.Services[0]
	568	+	mutate(&previous)
	569	+	active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	570	+	"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", Spec: previous},
	571	+	}}
	572	+
	573	+	expectDeployPreflight(state, runtime, images, secrets, rev)
	574	+	expectNetworkProvision(runtime, "app-blog", 1)
	575	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
	576	+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	577	+	var order []string
	578	+	expectRetireOld(state, runtime, &order, "c-old")
	579	+	expectSequentialCreate(state, runtime, &order, "c-new", 18080)
	580	+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
	581	+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
	582	+	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
	583	+
	584	+	svc := deployment.NewService(deployment.Deps{
	585	+	State: state, Runtime: runtime, Images: images, Secrets: secrets,
	586	+	Traffic: &orderedTraffic{events: &order},
	587	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	588	+	func(context.Context, string, string) (int, error) { return 200, nil },
	589	+	func(context.Context, string) error { return nil },
	590	+	))
	591	+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	592	+	require.NoError(t, err)
	593	+	assert.Equal(t, []string{"withdraw", "stop-old", "remove-old", "create-new", "start-new", "traffic"}, order[:6],
	594	+	"the predecessor is gone before the replacement is created")
	595	+	state.AssertNotCalled(t, "SaveRecoveryInhibition", mock.Anything, mock.Anything)
	596	+	})
	597	+	}
	598	+	}
	599	+
	600	+	// seedPendingRetirement seeds ACTIVE naming c-new and one retirement-pending
	601	+	// marker per given superseded container.
	602	+	func seedPendingRetirement(t testing.T, ctx context.Context, store appstate.Store, superseded ...string) {
	603	+	t.Helper()
	604	+	require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
	605	+	require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
	606	+	"web": {Container: "c-new", EffectiveRevision: "rev-1", Spec: overlapWebSpec(0), BackendBinds: map[int]int{8080: 18081}},
	607	+	}}))
	608	+	for _, id := range superseded {
	609	+	require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
	610	+	App: "blog", Service: "web", ContainerID: id, Reason: domain.AppInhibitRetirementPending,
	611	+	}))
	612	+	}
	613	+	}
	614	+
	615	+	func markerIDs(t testing.T, ctx context.Context, store appstate.Store) []string {
	616	+	t.Helper()
	617	+	markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
	618	+	require.NoError(t, err)
	619	+	ids := make([]string, 0, len(markers))
	620	+	for _, marker := range markers {
	621	+	ids = append(ids, marker.ContainerID)
	622	+	}
	623	+	return ids
	624	+	}
	625	+
	626	+	// TestReconcileRunning_DoesNotRetireWhenSuccessorIsNotVerifiedByThisPass is the
	627	+	// regression for a successor that was verified by an earlier pass but is
	628	+	// restarting in this one: it is withdrawn, not verified, and a final
	629	+	// inspection that happens to see it running must not retire the predecessor.
	630	+	func TestReconcileRunning_DoesNotRetireWhenSuccessorIsNotVerifiedByThisPass(t *testing.T) {
	631	+	ctx := context.Background()
	632	+	store := newTestStore(t)
	633	+	runtime := outmocks.NewMockContainerRuntime(t)
	634	+	images := outmocks.NewMockImageResolver(t)
	635	+	secrets := outmocks.NewMockSecretProvider(t)
	636	+	seedPendingRetirement(t, ctx, store, "c-old")
	637	+	traffic := &storeTraffic{store: store}
	638	+	svc := overlapService(store, runtime, images, secrets, traffic)
	639	+
	640	+	// Earlier pass: the successor is verified and routed, retirement fails.
	641	+	expectVerifiedRunning(runtime, "c-new", 18081)
	642	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	643	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
	644	+	require.Error(t, svc.ReconcileRunning(ctx))
	645	+
	646	+	// This pass: the successor is restarting, so it is withdrawn and not
	647	+	// verified. Any later inspection would see it running.
	648	+	runtime.ExpectedCalls, runtime.Calls = nil, nil
	649	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").
	650	+	Return(&domain.Container{ID: "c-new", Status: "restarting"}, nil).Once()
	651	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").
	652	+	Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: time.Now()}, nil).Maybe()
	653	+	require.NoError(t, svc.ReconcileRunning(ctx))
	654	+
	655	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
	656	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "the predecessor stays until a pass verifies the successor")
	657	+	}
	658	+
	659	+	// TestReconcileRunning_DoesNotRetireWhenSuccessorRestartedAfterVerification
	660	+	// covers the other half: a successor verified at one start time and seen with
	661	+	// another by the final inspection is a different execution, so unverified.
	662	+	func TestReconcileRunning_DoesNotRetireWhenSuccessorRestartedAfterVerification(t *testing.T) {
	663	+	ctx := context.Background()
	664	+	store := newTestStore(t)
	665	+	runtime := outmocks.NewMockContainerRuntime(t)
	666	+	images := outmocks.NewMockImageResolver(t)
	667	+	secrets := outmocks.NewMockSecretProvider(t)
	668	+	seedPendingRetirement(t, ctx, store, "c-old")
	669	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	670	+
	671	+	first := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
	672	+	restarted := first.Add(time.Minute)
	673	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: first}, nil).Twice()
	674	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: restarted}, nil).Once()
	675	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-new").Return("", false, nil)
	676	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	677	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil)
	678	+	require.NoError(t, svc.ReconcileRunning(ctx))
	679	+
	680	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
	681	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store))
	682	+	}
	683	+
	684	+	// TestReconcileRunning_RetiresStackedPredecessors proves several superseded
	685	+	// containers of one service (deploys that each left one behind) are all
	686	+	// retired once the successor is verified.
	687	+	func TestReconcileRunning_RetiresStackedPredecessors(t *testing.T) {
	688	+	ctx := context.Background()
	689	+	store := newTestStore(t)
	690	+	runtime := outmocks.NewMockContainerRuntime(t)
	691	+	images := outmocks.NewMockImageResolver(t)
	692	+	secrets := outmocks.NewMockSecretProvider(t)
	693	+	seedPendingRetirement(t, ctx, store, "c-old-1", "c-old-2")
	694	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	695	+
	696	+	expectVerifiedRunning(runtime, "c-new", 18081)
	697	+	for _, id := range []string{"c-old-1", "c-old-2"} {
	698	+	runtime.EXPECT().StopContainer(mock.Anything, id, domain.AppDefaultStopGrace).Return(nil).Once()
	699	+	runtime.EXPECT().RemoveContainer(mock.Anything, id, false).Return(nil).Once()
	700	+	}
	701	+	require.NoError(t, svc.ReconcileRunning(ctx))
	702	+	assert.Empty(t, markerIDs(t, ctx, store))
	703	+	}
	704	+
	705	+	// releaseFailingState wraps the real store and fails the first backend claim
	706	+	// releases, like a store write error would.
	707	+	type releaseFailingState struct {
	708	+	*appstate.Store
	709	+	failures int
	710	+	}
	711	+
	712	+	func (s *releaseFailingState) ReleaseBackendBinds(ctx context.Context, app, containerID string) error {
	713	+	if s.failures > 0 {
	714	+	s.failures--
	715	+	return assert.AnError
	716	+	}
	717	+	return s.Store.ReleaseBackendBinds(ctx, app, containerID)
	718	+	}
	719	+
	720	+	// TestReconcileRunning_KeepsMarkerUntilBackendClaimsAreReleased proves a
	721	+	// retirement whose claim release failed does not drop the durable marker: the
	722	+	// leak is reported by the pass and retried (retiring a gone container is
	723	+	// idempotent) until the claims are released.
	724	+	func TestReconcileRunning_KeepsMarkerUntilBackendClaimsAreReleased(t *testing.T) {
	725	+	ctx := context.Background()
	726	+	store := newTestStore(t)
	727	+	runtime := outmocks.NewMockContainerRuntime(t)
	728	+	images := outmocks.NewMockImageResolver(t)
	729	+	secrets := outmocks.NewMockSecretProvider(t)
	730	+	seedPendingRetirement(t, ctx, store, "c-old")
	731	+	state := &releaseFailingState{Store: store, failures: 1}
	732	+	traffic := &storeTraffic{store: store}
	733	+	svc := deployment.NewService(deployment.Deps{
	734	+	State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
	735	+	ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
	736	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	737	+	func(context.Context, string, string) (int, error) { return 200, nil },
	738	+	func(context.Context, string) error { return nil },
	739	+	))
	740	+
	741	+	expectVerifiedRunning(runtime, "c-new", 18081)
	742	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Twice()
	743	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	744	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
	745	+
	746	+	err := svc.ReconcileRunning(ctx)
	747	+	require.Error(t, err, "the leaked claims are reported, not discarded")
	748	+	assert.Contains(t, err.Error(), "release backend claims")
	749	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "the marker stays until the claims are released")
	750	+
	751	+	require.NoError(t, svc.ReconcileRunning(ctx), "not-found counts as gone, so the retry settles it")
	752	+	assert.Empty(t, markerIDs(t, ctx, store))
	753	+	}
	754	+
	755	+	// TestStop_RetiresPendingSupersededContainers proves a stopped app does not
	756	+	// keep a superseded predecessor running, and that one which cannot be retired
	757	+	// fails the stop and keeps its marker.
	758	+	func TestStop_RetiresPendingSupersededContainers(t *testing.T) {
	759	+	ctx := context.Background()
	760	+	store := newTestStore(t)
	761	+	runtime := outmocks.NewMockContainerRuntime(t)
	762	+	images := outmocks.NewMockImageResolver(t)
	763	+	secrets := outmocks.NewMockSecretProvider(t)
	764	+	seedPendingRetirement(t, ctx, store, "c-old")
	765	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	766	+
	767	+	runtime.EXPECT().StopContainer(mock.Anything, "c-new", mock.Anything).Return(nil).Twice()
	768	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", false).Return(nil).Twice()
	769	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Twice()
	770	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
	771	+
	772	+	result, err := svc.Stop(ctx, "blog", "")
	773	+	require.Error(t, err, "a predecessor that is still running fails the stop")
	774	+	assert.Equal(t, "failed", result.Services["web"].Result)
	775	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "its marker stays for the next attempt")
	776	+
	777	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	778	+	_, err = svc.Stop(ctx, "blog", "")
	779	+	require.NoError(t, err)
	780	+	assert.Empty(t, markerIDs(t, ctx, store))
	781	+	}
	782	+
	783	+	// TestRemove_FailsAndKeepsStateWhenAPendingPredecessorCannotBeRetired proves
	784	+	// Remove does not end the incarnation (which deletes the markers) while a
	785	+	// superseded container is still running.
	786	+	func TestRemove_FailsAndKeepsStateWhenAPendingPredecessorCannotBeRetired(t *testing.T) {
	787	+	ctx := context.Background()
	788	+	store := newTestStore(t)
	789	+	runtime := outmocks.NewMockContainerRuntime(t)
	790	+	images := outmocks.NewMockImageResolver(t)
	791	+	secrets := outmocks.NewMockSecretProvider(t)
	792	+	seedPendingRetirement(t, ctx, store, "c-old")
	793	+	svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
	794	+
	795	+	runtime.EXPECT().StopContainer(mock.Anything, "c-new", mock.Anything).Return(nil).Once()
	796	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", false).Return(nil).Once()
	797	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	798	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
	799	+
	800	+	_, err := svc.Remove(ctx, "blog", "")
	801	+	require.Error(t, err)
	802	+	assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "the marker survives so the container stays tracked")
	803	+	_, ok, loadErr := store.LoadActive(ctx, "blog")
	804	+	require.NoError(t, loadErr)
	805	+	assert.True(t, ok, "the incarnation is not ended")
	806	+	}
	807	+
	808	+	// inhibitionsFailingState wraps the real store and fails the load of the
	809	+	// recovery inhibitions, which is how Stop learns about pending predecessors.
	810	+	type inhibitionsFailingState struct {
	811	+	*appstate.Store
	812	+	}
	813	+
	814	+	func (s *inhibitionsFailingState) LoadRecoveryInhibitions(context.Context, string) ([]domain.AppRecoveryInhibition, error) {
	815	+	return nil, assert.AnError
	816	+	}
	817	+
	818	+	// TestStop_PendingRetirementLoadFailureIsAJournaledFailure proves an app-level
	819	+	// cleanup failure (the pending predecessors cannot even be listed) is not lost
	820	+	// in the per-service results: the operation is journaled as failed, so Stop
	821	+	// errors and a replay of the same key reads the same failure instead of a
	822	+	// success.
	823	+	func TestStop_PendingRetirementLoadFailureIsAJournaledFailure(t *testing.T) {
	824	+	ctx := context.Background()
	825	+	store := newTestStore(t)
	826	+	runtime := outmocks.NewMockContainerRuntime(t)
	827	+	seedPendingRetirement(t, ctx, store, "c-old")
	828	+	// The ACTIVE container is retired successfully before the load fails.
	829	+	runtime.EXPECT().StopContainer(mock.Anything, "c-new", mock.Anything).Return(nil).Once()
	830	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", false).Return(nil).Once()
	831	+	svc := deployment.NewService(deployment.Deps{
	832	+	State: &inhibitionsFailingState{Store: store}, Runtime: runtime, Traffic: &storeTraffic{store: store},
	833	+	}, zerowrap.Default())
	834	+
	835	+	result, err := svc.Stop(ctx, "blog", "key-stop")
	836	+	require.Error(t, err)
	837	+	assert.Equal(t, "deployed", result.Services["web"].Result, "the ACTIVE container really was stopped")
	838	+	require.NotEmpty(t, result.Warnings, "the app-level failure is surfaced on the result")
	839	+
	840	+	op, loadErr := store.LoadOperation(ctx, "blog", "key-stop")
	841	+	require.NoError(t, loadErr)
	842	+	assert.Equal(t, domain.AppOutcomeFailed, op.Outcome, "the journal records the failure, not the per-service success")
	843	+	step, found := opStep(op, "pending.retire")
	844	+	require.True(t, found)
	845	+	assert.Equal(t, domain.AppStepFailed, step.State)
	846	+	assert.NotEmpty(t, step.Error)
	847	+	require.NotEmpty(t, op.Warnings)
	848	+	assert.Contains(t, op.Warnings[0].Detail, "pending retirements unknown")
	849	+
	850	+	// Replaying the same key returns the same failure and runs nothing again.
	851	+	_, replayErr := svc.Stop(ctx, "blog", "key-stop")
	852	+	require.Error(t, replayErr)
	853	+	assert.ErrorIs(t, replayErr, domain.ErrAppStateConflict)
	854	+	assert.Contains(t, replayErr.Error(), domain.AppOutcomeFailed)
	855	+	runtime.AssertNumberOfCalls(t, "StopContainer", 1)
	856	+	}
	857	+
	858	+	// executionStartedAt is the start time the fakes report for a freshly started
	859	+	// container: deploy and restart observe it right after the start.
	860	+	var executionStartedAt = time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
	861	+
	862	+	// dropInspectExpectations removes the InspectContainer expectations so a test
	863	+	// can report a different execution after a deploy.
	864	+	func dropInspectExpectations(runtime *outmocks.MockContainerRuntime) {
	865	+	kept := runtime.ExpectedCalls[:0]
	866	+	for _, call := range runtime.ExpectedCalls {
	867	+	if call.Method != "InspectContainer" {
	868	+	kept = append(kept, call)
	869	+	}
	870	+	}
	871	+	runtime.ExpectedCalls = kept
	872	+	}
	873	+
	874	+	// TestReconcileRunning_DoesNotWithdrawAFreshlyDeployedGeneration is the
	875	+	// regression for a deployed generation being withdrawn by the first periodic
	876	+	// pass: the deploy verified and routed that execution, so the pass must treat
	877	+	// it as already seen. The traffic fake mutates ACTIVE like the real publisher,
	878	+	// so a withdrawal would drop the service's routed binds.
	879	+	func TestReconcileRunning_DoesNotWithdrawAFreshlyDeployedGeneration(t *testing.T) {
	880	+	ctx := context.Background()
	881	+	store := newTestStore(t)
	882	+	runtime := outmocks.NewMockContainerRuntime(t)
	883	+	images := outmocks.NewMockImageResolver(t)
	884	+	secrets := outmocks.NewMockSecretProvider(t)
	885	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	886	+	traffic := &storeTraffic{store: store}
	887	+	svc := overlapService(store, runtime, images, secrets, traffic)
	888	+
	889	+	overlapDeployMocks(runtime, images)
	890	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	891	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	892	+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	893	+	require.NoError(t, err)
	894	+	require.Equal(t, map[int]int{8080: 18081}, traffic.lastRoute("web"))
	895	+
	896	+	// The first periodic pass sees the same execution the deploy verified.
	897	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-new").Return("", false, nil)
	898	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	899	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil)
	900	+	require.NoError(t, svc.ReconcileRunning(ctx))
	901	+
	902	+	assert.Empty(t, traffic.withdrawals(), "the verified generation is never withdrawn")
	903	+	for i, routed := range traffic.allRoutes("web") {
	904	+	assert.Equal(t, map[int]int{8080: 18081}, routed, "graph %d kept routing the service", i)
	905	+	}
	906	+	active, _, err := store.LoadActive(ctx, "blog")
	907	+	require.NoError(t, err)
	908	+	assert.Equal(t, map[int]int{8080: 18081}, active.Services["web"].BackendBinds)
	909	+	}
	910	+
	911	+	// TestReconcileRunning_TreatsARestartedExecutionAfterDeployAsFresh proves the
	912	+	// recorded execution is the one observed before readiness: if the container
	913	+	// restarted natively afterwards, its start time differs and the pass withdraws
	914	+	// and verifies it again.
	915	+	func TestReconcileRunning_TreatsARestartedExecutionAfterDeployAsFresh(t *testing.T) {
	916	+	ctx := context.Background()
	917	+	store := newTestStore(t)
	918	+	runtime := outmocks.NewMockContainerRuntime(t)
	919	+	images := outmocks.NewMockImageResolver(t)
	920	+	secrets := outmocks.NewMockSecretProvider(t)
	921	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	922	+	traffic := &storeTraffic{store: store}
	923	+	svc := overlapService(store, runtime, images, secrets, traffic)
	924	+
	925	+	overlapDeployMocks(runtime, images)
	926	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	927	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	928	+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	929	+	require.NoError(t, err)
	930	+
	931	+	dropInspectExpectations(runtime)
	932	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(
	933	+	&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt.Add(time.Minute)}, nil)
	934	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-new").Return("", false, nil)
	935	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
	936	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18090, Protocol: domain.NetworkProtocolTCP}}, nil)
	937	+	require.NoError(t, svc.ReconcileRunning(ctx))
	938	+
	939	+	assert.Equal(t, []string{"blog/web"}, traffic.withdrawals(), "a different execution is withdrawn before it is verified")
	940	+	assert.Equal(t, map[int]int{8080: 18090}, traffic.lastRoute("web"), "and routed again on its verified binds")
	941	+	}
	942	+
	943	+	// TestReconcileRunning_DoesNotWithdrawARestartedGeneration covers the restart
	944	+	// path: an in-place restart verified and routed the new execution, so the
	945	+	// first periodic pass keeps it.
	946	+	func TestReconcileRunning_DoesNotWithdrawARestartedGeneration(t *testing.T) {
	947	+	ctx := context.Background()
	948	+	store := newTestStore(t)
	949	+	runtime := outmocks.NewMockContainerRuntime(t)
	950	+	images := outmocks.NewMockImageResolver(t)
	951	+	secrets := outmocks.NewMockSecretProvider(t)
	952	+	seedServingApp(t, ctx, store, overlapWebSpec(0))
	953	+	traffic := &storeTraffic{store: store}
	954	+	svc := overlapService(store, runtime, images, secrets, traffic)
	955	+
	956	+	restartedAt := executionStartedAt.Add(time.Hour)
	957	+	runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	958	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(
	959	+	&domain.Container{ID: "c-old", Status: "running", StartedAt: restartedAt}, nil)
	960	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
	961	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 17002, Protocol: domain.NetworkProtocolTCP}}, nil)
	962	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-old").Return("", false, nil)
	963	+	_, err := svc.Restart(ctx, "blog", "web", "")
	964	+	require.NoError(t, err)
	965	+	withdrawnByRestart := len(traffic.withdrawals())
	966	+
	967	+	require.NoError(t, svc.ReconcileRunning(ctx))
	968	+	assert.Len(t, traffic.withdrawals(), withdrawnByRestart, "the periodic pass withdraws nothing more")
	969	+	assert.Equal(t, map[int]int{8080: 17002}, traffic.lastRoute("web"))
	970	+	}
	971	+
	972	+	// TestReconcileRunning_RestoresARebuiltGenerationWhoseTrafficWasRejected is the
	973	+	// regression for a missing-container rebuild whose graph apply was rejected:
	974	+	// the rebuilt execution must not be recorded as verified, or the next periodic
	975	+	// pass sees an already verified generation whose binds match ACTIVE, skips the
	976	+	// publication, and leaves the service unrouted indefinitely.
	977	+	func TestReconcileRunning_RestoresARebuiltGenerationWhoseTrafficWasRejected(t *testing.T) {
	978	+	ctx := context.Background()
	979	+	store := newTestStore(t)
	980	+	runtime := outmocks.NewMockContainerRuntime(t)
	981	+	images := outmocks.NewMockImageResolver(t)
	982	+	secrets := outmocks.NewMockSecretProvider(t)
	983	+	seedReplaceableApp(t, ctx, store)
	984	+	traffic := &storeTraffic{store: store, failRebuild: 1}
	985	+	svc := deployment.NewService(deployment.Deps{
	986	+	State: store, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
	987	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	988	+	func(context.Context, string, string) (int, error) { return 200, nil },
	989	+	func(context.Context, string) error { return nil },
	990	+	))
	991	+
	992	+	// The recorded container is gone: restart rebuilds it from the pinned
	993	+	// ACTIVE digest, and the graph apply that follows is rejected.
	994	+	runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
	995	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
	996	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
	997	+	expectNetworkProvision(runtime, "app-blog", 1)
	998	+	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
	999	+	runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
	1000	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuilt").Return(
	1001	+	&domain.Container{ID: "c-rebuilt", Status: "running", StartedAt: executionStartedAt}, nil)
	1002	+	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
	1003	+	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil)
	1004	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-rebuilt").Return("", false, nil)
	1005	+
	1006	+	_, err := svc.Restart(ctx, "blog", "web", "op-restart")
	1007	+	require.Error(t, err, "a rejected graph is a failed restart")
	1008	+	require.Zero(t, traffic.rebuildCount(), "no graph was accepted")
	1009	+	active, _, loadErr := store.LoadActive(ctx, "blog")
	1010	+	require.NoError(t, loadErr)
	1011	+	require.Equal(t, "c-rebuilt", active.Services["web"].Container)
	1012	+
	1013	+	// The next periodic pass must verify and route the generation itself.
	1014	+	require.NoError(t, svc.ReconcileRunning(ctx))
	1015	+	assert.Equal(t, map[int]int{8080: 18082}, traffic.lastRoute("web"), "the rebuilt generation is routed again")
	1016	+	}

M internal/usecase/deployment/preflight_gate_test.go +1 -0


@@ -271,6 +271,7 @@ func TestDeploy_HoldsSharedGCBarrierAcrossResourceSelection(t *testing.T) {
271	271		expectNetworkProvision(runtime, "app-blog", 1)
272	272		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
273	273		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	274	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
274	275		state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil)
275	276		state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
276	277		state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)


@@ -271,6 +271,7 @@ func TestDeploy_HoldsSharedGCBarrierAcrossResourceSelection(t *testing.T) {
271	expectNetworkProvision(runtime, "app-blog", 1)	271		expectNetworkProvision(runtime, "app-blog", 1)
272	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()	272		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
273	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()	273		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
		274	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
274	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil)	275		state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil)
275	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)	276		state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
276	state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)	277		state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)

M internal/usecase/deployment/probes.go +42 -3


@@ -3,6 +3,7 @@ package deployment
3	3		import (
4	4		"bufio"
5	5		"context"
	6	+	"errors"
6	7		"fmt"
7	8		"io"
8	9		"net"

@@ -42,15 +43,53 @@ func httpGetOnce(ctx context.Context, probeURL, hostAuthority string) (int, erro
42	43		return resp.StatusCode, nil
43	44		}
44	45
45		-	// tcpDialOnce attempts one TCP connection.
	46	+	// tcpSettleWindow is how long an accepted TCP probe connection must stay open
	47	+	// (or deliver data) before the target counts as ready.
	48	+	const tcpSettleWindow = 300 * time.Millisecond
	49	+
	50	+	// errTCPProbeClosed reports a connection that was accepted and then closed
	51	+	// within the settle window.
	52	+	var errTCPProbeClosed = errors.New("connection closed right after accept")
	53	+
	54	+	// tcpDialOnce attempts one TCP readiness check against a published loopback
	55	+	// bind. A successful connect alone proves nothing: the host-side port relay
	56	+	// (rootless Docker, docker-proxy) accepts on behalf of the container and then
	57	+	// closes at once while the workload is not listening yet. So the connection
	58	+	// must also survive tcpSettleWindow: data from the workload (a banner) or a
	59	+	// connection still open at the end of the window means ready; EOF or a reset
	60	+	// means not ready.
46	61		func tcpDialOnce(ctx context.Context, addr string) error {
	62	+	return tcpProbeOnce(ctx, addr, tcpSettleWindow)
	63	+	}
	64	+
	65	+	func tcpProbeOnce(ctx context.Context, addr string, settle time.Duration) error {
47	66		dialer := &net.Dialer{Timeout: 2 * time.Second}
48	67		conn, err := dialer.DialContext(ctx, "tcp", addr)
49	68		if err != nil {
50	69		return err
51	70		}
52		-	_ = conn.Close()
53		-	return nil
	71	+	defer func() { _ = conn.Close() }()
	72	+	if err := conn.SetReadDeadline(time.Now().Add(settle)); err != nil {
	73	+	return fmt.Errorf("set probe read deadline: %w", err)
	74	+	}
	75	+	stop := context.AfterFunc(ctx, func() { _ = conn.SetReadDeadline(time.Now()) })
	76	+	defer stop()
	77	+	var buf [1]byte
	78	+	n, err := conn.Read(buf[:])
	79	+	if ctxErr := ctx.Err(); ctxErr != nil {
	80	+	return ctxErr
	81	+	}
	82	+	if n > 0 {
	83	+	return nil
	84	+	}
	85	+	var netErr net.Error
	86	+	if errors.As(err, &netErr) && netErr.Timeout() {
	87	+	return nil
	88	+	}
	89	+	if err == nil \|\| errors.Is(err, io.EOF) {
	90	+	return errTCPProbeClosed
	91	+	}
	92	+	return err
54	93		}
55	94
56	95		// scanForMarker reports whether the stream contains the marker.


@@ -3,6 +3,7 @@ package deployment
3		import (	3		import (
4		"bufio"	4		"bufio"
5		"context"	5		"context"
			6	+	"errors"
6		"fmt"	7		"fmt"
7		"io"	8		"io"
8		"net"	9		"net"

@@ -42,15 +43,53 @@ func httpGetOnce(ctx context.Context, probeURL, hostAuthority string) (int, erro
42		return resp.StatusCode, nil	43		return resp.StatusCode, nil
43		}	44		}
44			45
45	-	// tcpDialOnce attempts one TCP connection.	46	+	// tcpSettleWindow is how long an accepted TCP probe connection must stay open
			47	+	// (or deliver data) before the target counts as ready.
			48	+	const tcpSettleWindow = 300 * time.Millisecond
			49	+
			50	+	// errTCPProbeClosed reports a connection that was accepted and then closed
			51	+	// within the settle window.
			52	+	var errTCPProbeClosed = errors.New("connection closed right after accept")
			53	+
			54	+	// tcpDialOnce attempts one TCP readiness check against a published loopback
			55	+	// bind. A successful connect alone proves nothing: the host-side port relay
			56	+	// (rootless Docker, docker-proxy) accepts on behalf of the container and then
			57	+	// closes at once while the workload is not listening yet. So the connection
			58	+	// must also survive tcpSettleWindow: data from the workload (a banner) or a
			59	+	// connection still open at the end of the window means ready; EOF or a reset
			60	+	// means not ready.
46		func tcpDialOnce(ctx context.Context, addr string) error {	61		func tcpDialOnce(ctx context.Context, addr string) error {
			62	+	return tcpProbeOnce(ctx, addr, tcpSettleWindow)
			63	+	}
			64	+
			65	+	func tcpProbeOnce(ctx context.Context, addr string, settle time.Duration) error {
47		dialer := &net.Dialer{Timeout: 2 * time.Second}	66		dialer := &net.Dialer{Timeout: 2 * time.Second}
48		conn, err := dialer.DialContext(ctx, "tcp", addr)	67		conn, err := dialer.DialContext(ctx, "tcp", addr)
49		if err != nil {	68		if err != nil {
50		return err	69		return err
51		}	70		}
52	-	_ = conn.Close()	71	+	defer func() { _ = conn.Close() }()
53	-	return nil	72	+	if err := conn.SetReadDeadline(time.Now().Add(settle)); err != nil {
			73	+	return fmt.Errorf("set probe read deadline: %w", err)
			74	+	}
			75	+	stop := context.AfterFunc(ctx, func() { _ = conn.SetReadDeadline(time.Now()) })
			76	+	defer stop()
			77	+	var buf [1]byte
			78	+	n, err := conn.Read(buf[:])
			79	+	if ctxErr := ctx.Err(); ctxErr != nil {
			80	+	return ctxErr
			81	+	}
			82	+	if n > 0 {
			83	+	return nil
			84	+	}
			85	+	var netErr net.Error
			86	+	if errors.As(err, &netErr) && netErr.Timeout() {
			87	+	return nil
			88	+	}
			89	+	if err == nil \|\| errors.Is(err, io.EOF) {
			90	+	return errTCPProbeClosed
			91	+	}
			92	+	return err
54		}	93		}
55			94
56		// scanForMarker reports whether the stream contains the marker.	95		// scanForMarker reports whether the stream contains the marker.

A internal/usecase/deployment/probes_tcp_test.go +162 -0

@@ -0,0 +1,162 @@
	1	+	package deployment
	2	+
	3	+	import (
	4	+	"context"
	5	+	"net"
	6	+	"testing"
	7	+	"time"
	8	+
	9	+	"github.com/stretchr/testify/assert"
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	)
	14	+
	15	+	// serveTCP accepts connections on loopback and hands each to handle.
	16	+	func serveTCP(t *testing.T, handle func(net.Conn)) string {
	17	+	t.Helper()
	18	+	ln, err := net.Listen("tcp", "127.0.0.1:0")
	19	+	require.NoError(t, err)
	20	+	t.Cleanup(func() { _ = ln.Close() })
	21	+	go func() {
	22	+	for {
	23	+	conn, err := ln.Accept()
	24	+	if err != nil {
	25	+	return
	26	+	}
	27	+	go handle(conn)
	28	+	}
	29	+	}()
	30	+	return ln.Addr().String()
	31	+	}
	32	+
	33	+	// testSettle leaves a wide margin for the server goroutine to close first.
	34	+	const testSettle = 500 * time.Millisecond
	35	+
	36	+	func TestTCPProbeOnce_RelayAcceptThenCloseIsNotReady(t *testing.T) {
	37	+	// A host port relay with no listener behind it accepts and closes at once.
	38	+	addr := serveTCP(t, func(c net.Conn) { _ = c.Close() })
	39	+
	40	+	err := tcpProbeOnce(context.Background(), addr, testSettle)
	41	+
	42	+	require.ErrorIs(t, err, errTCPProbeClosed)
	43	+	}
	44	+
	45	+	func TestTCPProbeOnce_RelayResetIsNotReady(t *testing.T) {
	46	+	addr := serveTCP(t, func(c net.Conn) {
	47	+	if tc, ok := c.(*net.TCPConn); ok {
	48	+	_ = tc.SetLinger(0) // close with RST
	49	+	}
	50	+	_ = c.Close()
	51	+	})
	52	+
	53	+	err := tcpProbeOnce(context.Background(), addr, testSettle)
	54	+
	55	+	require.Error(t, err)
	56	+	}
	57	+
	58	+	func TestTCPProbeOnce_SilentListenerIsReady(t *testing.T) {
	59	+	// A workload that never speaks first (Postgres, Redis) keeps the connection open.
	60	+	release := make(chan struct{})
	61	+	t.Cleanup(func() { close(release) })
	62	+	addr := serveTCP(t, func(c net.Conn) {
	63	+	<-release
	64	+	_ = c.Close()
	65	+	})
	66	+
	67	+	start := time.Now()
	68	+	err := tcpProbeOnce(context.Background(), addr, testSettle)
	69	+
	70	+	require.NoError(t, err)
	71	+	assert.GreaterOrEqual(t, time.Since(start), testSettle)
	72	+	}
	73	+
	74	+	func TestTCPProbeOnce_BannerIsReadyImmediately(t *testing.T) {
	75	+	// A workload that speaks first (SSH, SMTP) is ready on its first byte.
	76	+	addr := serveTCP(t, func(c net.Conn) {
	77	+	_, _ = c.Write([]byte("SSH-2.0-test\r\n"))
	78	+	_ = c.Close()
	79	+	})
	80	+
	81	+	start := time.Now()
	82	+	err := tcpProbeOnce(context.Background(), addr, 5*time.Second)
	83	+
	84	+	require.NoError(t, err)
	85	+	assert.Less(t, time.Since(start), 2*time.Second)
	86	+	}
	87	+
	88	+	func TestTCPProbeOnce_NoListenerFails(t *testing.T) {
	89	+	ln, err := net.Listen("tcp", "127.0.0.1:0")
	90	+	require.NoError(t, err)
	91	+	addr := ln.Addr().String()
	92	+	require.NoError(t, ln.Close())
	93	+
	94	+	require.Error(t, tcpProbeOnce(context.Background(), addr, testSettle))
	95	+	}
	96	+
	97	+	func TestTCPProbeOnce_CanceledContextWinsOverData(t *testing.T) {
	98	+	addr := serveTCP(t, func(c net.Conn) {
	99	+	_, _ = c.Write([]byte("banner"))
	100	+	_ = c.Close()
	101	+	})
	102	+	ctx, cancel := context.WithCancel(context.Background())
	103	+	cancel()
	104	+
	105	+	require.Error(t, tcpProbeOnce(ctx, addr, testSettle))
	106	+	}
	107	+
	108	+	func TestWaitTCPReady_SettleAfterBudgetIsNotReady(t *testing.T) {
	109	+	// Each attempt settles only after the readiness budget expired: the wait
	110	+	// must report a timeout instead of accepting the late success.
	111	+	slowDial := func(ctx context.Context, _ string) error {
	112	+	select {
	113	+	case <-ctx.Done():
	114	+	case <-time.After(400 * time.Millisecond):
	115	+	}
	116	+	return nil
	117	+	}
	118	+	deps := NewTestProbeDeps(nil, nil, slowDial)
	119	+	spec := domain.AppService{
	120	+	Name: "relay",
	121	+	TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 9000}},
	122	+	Readiness: domain.AppReadiness{Type: domain.AppReadinessTCP, Timeout: 200 * time.Millisecond},
	123	+	}
	124	+
	125	+	start := time.Now()
	126	+	err := waitTCPReadyWithDeps(context.Background(), deps, spec, map[int]int{9000: 40000})
	127	+
	128	+	require.ErrorContains(t, err, "TCP readiness timeout")
	129	+	assert.Less(t, time.Since(start), 5*time.Second)
	130	+	}
	131	+
	132	+	func TestWaitTCPReady_CallerCancelIsReturned(t *testing.T) {
	133	+	deps := NewTestProbeDeps(nil, nil, func(context.Context, string) error { return errTCPProbeClosed })
	134	+	spec := domain.AppService{
	135	+	Name: "relay",
	136	+	TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 9000}},
	137	+	Readiness: domain.AppReadiness{Type: domain.AppReadinessTCP, Timeout: 10 * time.Second},
	138	+	}
	139	+	ctx, cancel := context.WithCancel(context.Background())
	140	+	cancel()
	141	+
	142	+	err := waitTCPReadyWithDeps(ctx, deps, spec, map[int]int{9000: 40000})
	143	+
	144	+	require.ErrorIs(t, err, context.Canceled)
	145	+	}
	146	+
	147	+	func TestTCPProbeOnce_ContextCancelStopsTheSettleWait(t *testing.T) {
	148	+	release := make(chan struct{})
	149	+	t.Cleanup(func() { close(release) })
	150	+	addr := serveTCP(t, func(c net.Conn) {
	151	+	<-release
	152	+	_ = c.Close()
	153	+	})
	154	+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
	155	+	defer cancel()
	156	+
	157	+	start := time.Now()
	158	+	err := tcpProbeOnce(ctx, addr, 10*time.Second)
	159	+
	160	+	require.ErrorIs(t, err, context.DeadlineExceeded)
	161	+	assert.Less(t, time.Since(start), 5*time.Second)
	162	+	}

@@ -0,0 +1,162 @@
	1	+	package deployment
	2	+
	3	+	import (
	4	+	"context"
	5	+	"net"
	6	+	"testing"
	7	+	"time"
	8	+
	9	+	"github.com/stretchr/testify/assert"
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	)
	14	+
	15	+	// serveTCP accepts connections on loopback and hands each to handle.
	16	+	func serveTCP(t *testing.T, handle func(net.Conn)) string {
	17	+	t.Helper()
	18	+	ln, err := net.Listen("tcp", "127.0.0.1:0")
	19	+	require.NoError(t, err)
	20	+	t.Cleanup(func() { _ = ln.Close() })
	21	+	go func() {
	22	+	for {
	23	+	conn, err := ln.Accept()
	24	+	if err != nil {
	25	+	return
	26	+	}
	27	+	go handle(conn)
	28	+	}
	29	+	}()
	30	+	return ln.Addr().String()
	31	+	}
	32	+
	33	+	// testSettle leaves a wide margin for the server goroutine to close first.
	34	+	const testSettle = 500 * time.Millisecond
	35	+
	36	+	func TestTCPProbeOnce_RelayAcceptThenCloseIsNotReady(t *testing.T) {
	37	+	// A host port relay with no listener behind it accepts and closes at once.
	38	+	addr := serveTCP(t, func(c net.Conn) { _ = c.Close() })
	39	+
	40	+	err := tcpProbeOnce(context.Background(), addr, testSettle)
	41	+
	42	+	require.ErrorIs(t, err, errTCPProbeClosed)
	43	+	}
	44	+
	45	+	func TestTCPProbeOnce_RelayResetIsNotReady(t *testing.T) {
	46	+	addr := serveTCP(t, func(c net.Conn) {
	47	+	if tc, ok := c.(*net.TCPConn); ok {
	48	+	_ = tc.SetLinger(0) // close with RST
	49	+	}
	50	+	_ = c.Close()
	51	+	})
	52	+
	53	+	err := tcpProbeOnce(context.Background(), addr, testSettle)
	54	+
	55	+	require.Error(t, err)
	56	+	}
	57	+
	58	+	func TestTCPProbeOnce_SilentListenerIsReady(t *testing.T) {
	59	+	// A workload that never speaks first (Postgres, Redis) keeps the connection open.
	60	+	release := make(chan struct{})
	61	+	t.Cleanup(func() { close(release) })
	62	+	addr := serveTCP(t, func(c net.Conn) {
	63	+	<-release
	64	+	_ = c.Close()
	65	+	})
	66	+
	67	+	start := time.Now()
	68	+	err := tcpProbeOnce(context.Background(), addr, testSettle)
	69	+
	70	+	require.NoError(t, err)
	71	+	assert.GreaterOrEqual(t, time.Since(start), testSettle)
	72	+	}
	73	+
	74	+	func TestTCPProbeOnce_BannerIsReadyImmediately(t *testing.T) {
	75	+	// A workload that speaks first (SSH, SMTP) is ready on its first byte.
	76	+	addr := serveTCP(t, func(c net.Conn) {
	77	+	_, _ = c.Write([]byte("SSH-2.0-test\r\n"))
	78	+	_ = c.Close()
	79	+	})
	80	+
	81	+	start := time.Now()
	82	+	err := tcpProbeOnce(context.Background(), addr, 5*time.Second)
	83	+
	84	+	require.NoError(t, err)
	85	+	assert.Less(t, time.Since(start), 2*time.Second)
	86	+	}
	87	+
	88	+	func TestTCPProbeOnce_NoListenerFails(t *testing.T) {
	89	+	ln, err := net.Listen("tcp", "127.0.0.1:0")
	90	+	require.NoError(t, err)
	91	+	addr := ln.Addr().String()
	92	+	require.NoError(t, ln.Close())
	93	+
	94	+	require.Error(t, tcpProbeOnce(context.Background(), addr, testSettle))
	95	+	}
	96	+
	97	+	func TestTCPProbeOnce_CanceledContextWinsOverData(t *testing.T) {
	98	+	addr := serveTCP(t, func(c net.Conn) {
	99	+	_, _ = c.Write([]byte("banner"))
	100	+	_ = c.Close()
	101	+	})
	102	+	ctx, cancel := context.WithCancel(context.Background())
	103	+	cancel()
	104	+
	105	+	require.Error(t, tcpProbeOnce(ctx, addr, testSettle))
	106	+	}
	107	+
	108	+	func TestWaitTCPReady_SettleAfterBudgetIsNotReady(t *testing.T) {
	109	+	// Each attempt settles only after the readiness budget expired: the wait
	110	+	// must report a timeout instead of accepting the late success.
	111	+	slowDial := func(ctx context.Context, _ string) error {
	112	+	select {
	113	+	case <-ctx.Done():
	114	+	case <-time.After(400 * time.Millisecond):
	115	+	}
	116	+	return nil
	117	+	}
	118	+	deps := NewTestProbeDeps(nil, nil, slowDial)
	119	+	spec := domain.AppService{
	120	+	Name: "relay",
	121	+	TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 9000}},
	122	+	Readiness: domain.AppReadiness{Type: domain.AppReadinessTCP, Timeout: 200 * time.Millisecond},
	123	+	}
	124	+
	125	+	start := time.Now()
	126	+	err := waitTCPReadyWithDeps(context.Background(), deps, spec, map[int]int{9000: 40000})
	127	+
	128	+	require.ErrorContains(t, err, "TCP readiness timeout")
	129	+	assert.Less(t, time.Since(start), 5*time.Second)
	130	+	}
	131	+
	132	+	func TestWaitTCPReady_CallerCancelIsReturned(t *testing.T) {
	133	+	deps := NewTestProbeDeps(nil, nil, func(context.Context, string) error { return errTCPProbeClosed })
	134	+	spec := domain.AppService{
	135	+	Name: "relay",
	136	+	TCP: []domain.AppTCPInterface{{Entrypoint: "tcp", Port: 9000}},
	137	+	Readiness: domain.AppReadiness{Type: domain.AppReadinessTCP, Timeout: 10 * time.Second},
	138	+	}
	139	+	ctx, cancel := context.WithCancel(context.Background())
	140	+	cancel()
	141	+
	142	+	err := waitTCPReadyWithDeps(ctx, deps, spec, map[int]int{9000: 40000})
	143	+
	144	+	require.ErrorIs(t, err, context.Canceled)
	145	+	}
	146	+
	147	+	func TestTCPProbeOnce_ContextCancelStopsTheSettleWait(t *testing.T) {
	148	+	release := make(chan struct{})
	149	+	t.Cleanup(func() { close(release) })
	150	+	addr := serveTCP(t, func(c net.Conn) {
	151	+	<-release
	152	+	_ = c.Close()
	153	+	})
	154	+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
	155	+	defer cancel()
	156	+
	157	+	start := time.Now()
	158	+	err := tcpProbeOnce(ctx, addr, 10*time.Second)
	159	+
	160	+	require.ErrorIs(t, err, context.DeadlineExceeded)
	161	+	assert.Less(t, time.Since(start), 5*time.Second)
	162	+	}

M internal/usecase/deployment/publication_internal_test.go +2 -0


@@ -39,6 +39,7 @@ func TestVerifyRunningService_NotReadyWithdrawsAndClearsBinds(t *testing.T) {
39	39		Spec: readySpec(),
40	40		BackendBinds: map[int]int{8080: 32771},
41	41		}
	42	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
42	43		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).Return(
43	44		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
44	45		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()

@@ -71,6 +72,7 @@ func TestVerifyRunningService_ReadyPublishesBinds(t *testing.T) {
71	72		runtime := outmocks.NewMockContainerRuntime(t)
72	73		traffic := &stubTraffic{}
73	74		eff := domain.AppEffectiveService{Container: "c-1", EffectiveRevision: "rev-1", Spec: readySpec()}
	75	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
74	76		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).Return(
75	77		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
76	78		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()


@@ -39,6 +39,7 @@ func TestVerifyRunningService_NotReadyWithdrawsAndClearsBinds(t *testing.T) {
39	Spec: readySpec(),	39		Spec: readySpec(),
40	BackendBinds: map[int]int{8080: 32771},	40		BackendBinds: map[int]int{8080: 32771},
41	}	41		}
		42	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
42	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).Return(	43		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).Return(
43	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()	44		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
44	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()	45		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()

@@ -71,6 +72,7 @@ func TestVerifyRunningService_ReadyPublishesBinds(t *testing.T) {
71	runtime := outmocks.NewMockContainerRuntime(t)	72		runtime := outmocks.NewMockContainerRuntime(t)
72	traffic := &stubTraffic{}	73		traffic := &stubTraffic{}
73	eff := domain.AppEffectiveService{Container: "c-1", EffectiveRevision: "rev-1", Spec: readySpec()}	74		eff := domain.AppEffectiveService{Container: "c-1", EffectiveRevision: "rev-1", Spec: readySpec()}
		75	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
74	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).Return(	76		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).Return(
75	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()	77		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
76	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()	78		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()

M internal/usecase/deployment/readiness.go +14 -8


@@ -344,20 +344,26 @@ func waitTCPReadyWithDeps(ctx context.Context, deps ProbeDeps, spec domain.AppSe
344	344		return err
345	345		}
346	346		addr := net.JoinHostPort("127.0.0.1", itoa(port))
347		-	deadline := time.Now().Add(timeout)
348		-	for {
	347	+	// Every attempt and retry wait runs under the readiness budget, so a
	348	+	// connection that only settles after the budget expired is never accepted.
	349	+	attemptCtx, cancel := context.WithTimeout(ctx, timeout)
	350	+	defer cancel()
	351	+	timedOut := func() error {
349	352		if err := ctx.Err(); err != nil {
350	353		return err
351	354		}
352		-	if err := deps.tcpDial(ctx, addr); err == nil {
353		-	return nil
	355	+	return fmt.Errorf("deployment: TCP readiness timeout after %s: %s not reachable", timeout, addr)
	356	+	}
	357	+	for {
	358	+	if attemptCtx.Err() != nil {
	359	+	return timedOut()
354	360		}
355		-	if time.Now().After(deadline) {
356		-	return fmt.Errorf("deployment: TCP readiness timeout after %s: %s not reachable", timeout, addr)
	361	+	if err := deps.tcpDial(attemptCtx, addr); err == nil && attemptCtx.Err() == nil {
	362	+	return nil
357	363		}
358	364		select {
359		-	case <-ctx.Done():
360		-	return ctx.Err()
	365	+	case <-attemptCtx.Done():
	366	+	return timedOut()
361	367		case <-time.After(500 * time.Millisecond):
362	368		}
363	369		}

 		return err
 	}
 	addr := net.JoinHostPort("127.0.0.1", itoa(port))
-	deadline := time.Now().Add(timeout)
+	// Every attempt and retry wait runs under the readiness budget, so a
-	for {
+	// connection that only settles after the budget expired is never accepted.
+	attemptCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	timedOut := func() error {
 		if err := ctx.Err(); err != nil {
 			return err
 		}
-		if err := deps.tcpDial(ctx, addr); err == nil {
+		return fmt.Errorf("deployment: TCP readiness timeout after %s: %s not reachable", timeout, addr)
-			return nil
+	}
+	for {
+		if attemptCtx.Err() != nil {
+			return timedOut()
 		}
-		if time.Now().After(deadline) {
+		if err := deps.tcpDial(attemptCtx, addr); err == nil && attemptCtx.Err() == nil {
-			return fmt.Errorf("deployment: TCP readiness timeout after %s: %s not reachable", timeout, addr)
+			return nil
 		}
 		select {
-		case <-ctx.Done():
+		case <-attemptCtx.Done():
-			return ctx.Err()
+			return timedOut()
 		case <-time.After(500 * time.Millisecond):
 		}
 	}

M internal/usecase/deployment/recovery_integration_test.go +2 -0


@@ -211,6 +211,7 @@ func TestRemove_InhibitsBeforeRuntimeWithdrawal(t *testing.T) {
211	211		state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
212	212		// Ownership is read while it is still live; the name-only record owns
213	213		// no network, so nothing is reclaimed.
	214	+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
214	215		state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
215	216		// The incarnation is retired atomically after the workload is gone.
216	217		state.EXPECT().RetireApp(mock.Anything, "blog").RunAndReturn(func(context.Context, string) error {

@@ -417,6 +418,7 @@ func TestRemovalOfMissingContainerIsIdempotent(t *testing.T) {
417	418		// the disappeared container are released.
418	419		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
419	420		state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
	421	+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
420	422		state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
421	423		state.EXPECT().RetireApp(mock.Anything, "blog").Return(nil).Once()
422	424


@@ -211,6 +211,7 @@ func TestRemove_InhibitsBeforeRuntimeWithdrawal(t *testing.T) {
211	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()	211		state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
212	// Ownership is read while it is still live; the name-only record owns	212		// Ownership is read while it is still live; the name-only record owns
213	// no network, so nothing is reclaimed.	213		// no network, so nothing is reclaimed.
		214	+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
214	state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()	215		state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
215	// The incarnation is retired atomically after the workload is gone.	216		// The incarnation is retired atomically after the workload is gone.
216	state.EXPECT().RetireApp(mock.Anything, "blog").RunAndReturn(func(context.Context, string) error {	217		state.EXPECT().RetireApp(mock.Anything, "blog").RunAndReturn(func(context.Context, string) error {

@@ -417,6 +418,7 @@ func TestRemovalOfMissingContainerIsIdempotent(t *testing.T) {
417	// the disappeared container are released.	418		// the disappeared container are released.
418	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()	419		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
419	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()	420		state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
		421	+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
420	state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()	422		state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
421	state.EXPECT().RetireApp(mock.Anything, "blog").Return(nil).Once()	423		state.EXPECT().RetireApp(mock.Anything, "blog").Return(nil).Once()
422		424

M internal/usecase/deployment/restart_inplace_test.go +2 -0


@@ -78,6 +78,7 @@ func TestRestart_InPlaceWithdrawsRestartsVerifiesAndRepublishes(t *testing.T) {
78	78		recordEvent(&order, "restart")
79	79		return nil
80	80		}).Once()
	81	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
81	82		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
82	83		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
83	84		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()

@@ -135,6 +136,7 @@ func TestRestart_ReadinessFailureLeavesTheRestartedContainerWithdrawn(t *testing
135	136		state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
136	137		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
137	138		runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	139	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
138	140		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
139	141		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
140	142		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()


@@ -78,6 +78,7 @@ func TestRestart_InPlaceWithdrawsRestartsVerifiesAndRepublishes(t *testing.T) {
78	recordEvent(&order, "restart")	78		recordEvent(&order, "restart")
79	return nil	79		return nil
80	}).Once()	80		}).Once()
		81	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
81	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(	82		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
82	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()	83		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
83	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()	84		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()

@@ -135,6 +136,7 @@ func TestRestart_ReadinessFailureLeavesTheRestartedContainerWithdrawn(t *testing
135	state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)	136		state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
136	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()	137		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
137	runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()	138		runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
		139	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
138	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(	140		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
139	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()	141		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
140	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()	142		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()

M internal/usecase/deployment/retirement.go +19 -2


@@ -50,6 +50,14 @@ type retireOptions struct {
50	50		// published, because recovery must not recreate the replaced writer
51	51		// from the still-active old revision.
52	52		ClearInhibition bool
	53	+	// RetryOnLeak keeps the inhibition when the container is gone but its
	54	+	// backend claims could not be released, so a durable retirement marker is
	55	+	// retried (retiring a gone container is idempotent: not-found counts as
	56	+	// gone) instead of leaking the claims untracked. Callers that clear an
	57	+	// inhibition for a container that must simply never be revived (stop,
	58	+	// remove) leave it unset: a lingering marker there would refuse the
	59	+	// next start.
	60	+	RetryOnLeak bool
53	61		}
54	62
55	63		// containerRetirement is the outcome of one exact-container retirement.

@@ -59,6 +67,10 @@ type containerRetirement struct {
59	67		// Gone is true only when the runtime confirmed the container no
60	68		// longer exists. Not-found counts as confirmation.
61	69		Gone bool
	70	+	// Settled is true when the container is gone AND its backend claims were
	71	+	// released. Only a settled retirement may drop a durable marker; a gone
	72	+	// container whose claims leaked is retried, which is idempotent.
	73	+	Settled bool
62	74		// Warnings are bounded, log-free leftovers for operator action.
63	75		Warnings []CleanupWarning
64	76		}

@@ -74,6 +86,7 @@ func (s *Service) retireContainer(ctx context.Context, app string, opts retireOp
74	86		result := containerRetirement{ContainerID: containerID}
75	87		if containerID == "" {
76	88		result.Gone = true
	89	+	result.Settled = true
77	90		return result
78	91		}
79	92		if err := s.stopForRetirement(ctx, opts, containerID); err != nil {

@@ -89,7 +102,9 @@ func (s *Service) retireContainer(ctx context.Context, app string, opts retireOp
89	102		return result
90	103		}
91	104		result.Gone = true
	105	+	released := true
92	106		if err := s.deps.State.ReleaseBackendBinds(ctx, app, containerID); err != nil {
	107	+	released = false
93	108		result.Warnings = append(result.Warnings, CleanupWarning{
94	109		Service: opts.Service, Leftover: containerID, Detail: "release backend claims: " + err.Error(),
95	110		})

@@ -105,9 +120,11 @@ func (s *Service) retireContainer(ctx context.Context, app string, opts retireOp
105	120		Service: opts.Service, Leftover: containerID, Detail: "secret files cleanup failed",
106	121		})
107	122		}
108		-	if opts.ClearInhibition {
	123	+	result.Settled = released
	124	+	if opts.ClearInhibition && (released \|\| !opts.RetryOnLeak) {
109	125		// Confirmed gone: the marker has nothing left to protect, and a
110		-	// stale record would only confuse the next recreation.
	126	+	// stale record would only confuse the next recreation. With
	127	+	// RetryOnLeak it also waits for the claims to be released.
111	128		if err := s.clearRecoveryInhibition(ctx, app, opts.Service, containerID); err != nil {
112	129		result.Warnings = append(result.Warnings, CleanupWarning{
113	130		Service: opts.Service, Leftover: containerID, Detail: "clear recovery inhibition: " + err.Error(),

 	// published, because recovery must not recreate the replaced writer
 	// from the still-active old revision.
 	ClearInhibition bool
+	// RetryOnLeak keeps the inhibition when the container is gone but its
+	// backend claims could not be released, so a durable retirement marker is
+	// retried (retiring a gone container is idempotent: not-found counts as
+	// gone) instead of leaking the claims untracked. Callers that clear an
+	// inhibition for a container that must simply never be revived (stop,
+	// remove) leave it unset: a lingering marker there would refuse the
+	// next start.
+	RetryOnLeak bool
 }
 // containerRetirement is the outcome of one exact-container retirement.
 	// Gone is true only when the runtime confirmed the container no
 	// longer exists. Not-found counts as confirmation.
 	Gone bool
+	// Settled is true when the container is gone AND its backend claims were
+	// released. Only a settled retirement may drop a durable marker; a gone
+	// container whose claims leaked is retried, which is idempotent.
+	Settled bool
 	// Warnings are bounded, log-free leftovers for operator action.
 	Warnings []CleanupWarning
 }
 	result := containerRetirement{ContainerID: containerID}
 	if containerID == "" {
 		result.Gone = true
+		result.Settled = true
 		return result
 	}
 	if err := s.stopForRetirement(ctx, opts, containerID); err != nil {
 		return result
 	}
 	result.Gone = true
+	released := true
 	if err := s.deps.State.ReleaseBackendBinds(ctx, app, containerID); err != nil {
+		released = false
 		result.Warnings = append(result.Warnings, CleanupWarning{
 			Service: opts.Service, Leftover: containerID, Detail: "release backend claims: " + err.Error(),
 		})
 			Service: opts.Service, Leftover: containerID, Detail: "secret files cleanup failed",
 		})
 	}
-	if opts.ClearInhibition {
+	result.Settled = released
+	if opts.ClearInhibition && (released || !opts.RetryOnLeak) {
 		// Confirmed gone: the marker has nothing left to protect, and a
-		// stale record would only confuse the next recreation.
+		// stale record would only confuse the next recreation. With
+		// RetryOnLeak it also waits for the claims to be released.
 		if err := s.clearRecoveryInhibition(ctx, app, opts.Service, containerID); err != nil {
 			result.Warnings = append(result.Warnings, CleanupWarning{
 				Service: opts.Service, Leftover: containerID, Detail: "clear recovery inhibition: " + err.Error(),

M internal/usecase/deployment/retirement_test.go +7 -1


@@ -39,6 +39,7 @@ func TestLifecycleVerbsPassTheEffectiveStopGrace(t *testing.T) {
39	39		state.EXPECT().Recover(mock.Anything).Return(nil)
40	40		state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
41	41		state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
	42	+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
42	43		runtime.EXPECT().StopContainer(mock.Anything, "c-1", grace).Return(nil).Once()
43	44		runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
44	45		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()

@@ -58,6 +59,7 @@ func TestLifecycleVerbsPassTheEffectiveStopGrace(t *testing.T) {
58	59		state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
59	60		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
60	61		runtime.EXPECT().RestartContainer(mock.Anything, "c-1", grace).Return(nil).Once()
	62	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
61	63		state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
62	64
63	65		svc := deployment.NewService(deployment.Deps{State: state, Runtime: runtime}, zerowrap.Default())

@@ -77,6 +79,7 @@ func TestLifecycleVerbsPassTheEffectiveStopGrace(t *testing.T) {
77	79		runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
78	80		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
79	81		state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
	82	+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
80	83		state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
81	84		state.EXPECT().RetireApp(mock.Anything, "blog").Return(nil).Once()
82	85		state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)

@@ -97,6 +100,7 @@ func TestStopWithoutDeclaredGraceUsesTheAppDefault(t *testing.T) {
97	100		state.EXPECT().Recover(mock.Anything).Return(nil)
98	101		state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
99	102		state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
	103	+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
100	104		runtime.EXPECT().StopContainer(mock.Anything, "c-1", domain.AppDefaultStopGrace).Return(nil).Once()
101	105		runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
102	106		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()

@@ -140,7 +144,7 @@ func TestRemove_FailedRetirementKeepsClaimsAndActiveState(t *testing.T) {
140	144		func TestDeploy_RetirementFailureBlocksReplacement(t *testing.T) {
141	145		ctx := context.Background()
142	146		state, runtime, images, secrets := mockDeps(t)
143		-	rev := mockRevision()
	147	+	rev := statefulMockRevision()
144	148
145	149		oldActive := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
146	150		"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", Spec: graceSpec(time.Second)},

@@ -156,6 +160,8 @@ func TestDeploy_RetirementFailureBlocksReplacement(t *testing.T) {
156	160		state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
157	161		state.EXPECT().LoadActive(mock.Anything, "blog").Return(oldActive, true, nil)
158	162		state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	163	+	runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil)
	164	+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
159	165		// The superseded container is retired with its own effective grace, and
160	166		// its removal fails: the replacement must not start.
161	167		runtime.EXPECT().StopContainer(mock.Anything, "c-old", time.Second).Return(nil).Once()

 		state.EXPECT().Recover(mock.Anything).Return(nil)
 		state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
 		state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
+		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
 		runtime.EXPECT().StopContainer(mock.Anything, "c-1", grace).Return(nil).Once()
 		runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
 		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
 		state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
 		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
 		runtime.EXPECT().RestartContainer(mock.Anything, "c-1", grace).Return(nil).Once()
+		runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 		state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
 		svc := deployment.NewService(deployment.Deps{State: state, Runtime: runtime}, zerowrap.Default())
 		runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
 		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
 		state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
+		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
 		state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
 		state.EXPECT().RetireApp(mock.Anything, "blog").Return(nil).Once()
 		state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
 	state.EXPECT().Recover(mock.Anything).Return(nil)
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
 	state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
 	runtime.EXPECT().StopContainer(mock.Anything, "c-1", domain.AppDefaultStopGrace).Return(nil).Once()
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
 	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
 func TestDeploy_RetirementFailureBlocksReplacement(t *testing.T) {
 	ctx := context.Background()
 	state, runtime, images, secrets := mockDeps(t)
-	rev := mockRevision()
+	rev := statefulMockRevision()
 	oldActive := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
 		"web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", Spec: graceSpec(time.Second)},
 	state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(oldActive, true, nil)
 	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
+	runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil)
+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
 	// The superseded container is retired with its own effective grace, and
 	// its removal fails: the replacement must not start.
 	runtime.EXPECT().StopContainer(mock.Anything, "c-old", time.Second).Return(nil).Once()

M internal/usecase/deployment/sequential_replacement_test.go +126 -22


@@ -112,11 +112,22 @@ func expectSequentialCreate(
112	112		recordEvent(events, "start-new")
113	113		return nil
114	114		}).Once()
	115	+	// Deploy observes the execution start right after the start.
	116	+	runtime.EXPECT().InspectContainer(mock.Anything, containerID).Return(
	117	+	&domain.Container{ID: containerID, Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
115	118		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, containerID, mock.Anything).Return(
116	119		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: hostPort, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
117	120		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
118	121		}
119	122
	123	+	// expectRetirementMarker wires the durable retirement-pending marker an
	124	+	// overlap replacement records for the superseded container before the switch.
	125	+	func expectRetirementMarker(state *outmocks.MockAppState, containerID string) {
	126	+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.MatchedBy(func(i domain.AppRecoveryInhibition) bool {
	127	+	return i.ContainerID == containerID && i.Reason == domain.AppInhibitRetirementPending
	128	+	})).Return(nil).Once()
	129	+	}
	130	+
120	131		// expectRetireOld wires the confirmed retirement of the superseded container.
121	132		func expectRetireOld(
122	133		state *outmocks.MockAppState,

@@ -137,12 +148,13 @@ func expectRetireOld(
137	148		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", containerID).Return(nil).Once()
138	149		}
139	150
140		-	// TestDeploy_SequentialReplacementNeverOverlapsGenerations proves the
141		-	// replacement order for a public HTTP service without volumes: withdrawal is
142		-	// confirmed first, the superseded container is stopped and removed before the
143		-	// replacement is created, readiness runs before ACTIVE is published, and the
144		-	// graph is applied last.
145		-	func TestDeploy_SequentialReplacementNeverOverlapsGenerations(t *testing.T) {
	151	+	// TestDeploy_StatelessReplacementOverlapsAndSwitchesBeforeRetiring proves the
	152	+	// replacement order for a public HTTP service without volumes, binds, or
	153	+	// devices: the candidate is created and ready while the superseded container
	154	+	// keeps serving, ACTIVE and the proxy graph are switched to it, and only then
	155	+	// is the superseded container stopped and removed. The old container is never
	156	+	// withdrawn from traffic and never inhibited.
	157	+	func TestDeploy_StatelessReplacementOverlapsAndSwitchesBeforeRetiring(t *testing.T) {
146	158		ctx := context.Background()
147	159		state, runtime, images, secrets := mockDeps(t)
148	160		rev := mockRevision()

@@ -153,16 +165,20 @@ func TestDeploy_SequentialReplacementNeverOverlapsGenerations(t *testing.T) {
153	165		state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
154	166
155	167		var order []string
156		-	expectRetireOld(state, runtime, &order, "c-old")
157	168		expectSequentialCreate(state, runtime, &order, "c-new", 18080)
158	169		state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
159		-	state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(
	170	+	state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(active domain.AppActive) bool {
	171	+	return active.Services["web"].Container == "c-new" && active.Services["web"].BackendBinds[8080] == 18080
	172	+	})).RunAndReturn(
160	173		func(context.Context, domain.AppActive) error {
161	174		recordEvent(&order, "publish-active")
162	175		return nil
163	176		}).Once()
164	177		state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
	178	+	expectRetirementMarker(state, "c-old")
	179	+	// Confirmed retirement drops the marker.
165	180		state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
	181	+	expectRetireOld(state, runtime, &order, "c-old")
166	182
167	183		var probedURL string
168	184		traffic := &orderedTraffic{events: &order}

@@ -183,8 +199,84 @@ func TestDeploy_SequentialReplacementNeverOverlapsGenerations(t *testing.T) {
183	199		assert.Equal(t, "deployed", result.Services["web"].Result)
184	200		assert.Contains(t, probedURL, "127.0.0.1:18080", "readiness dials the replacement's fresh loopback bind")
185	201		assert.Equal(t,
186		-	[]string{"withdraw", "stop-old", "remove-old", "create-new", "start-new", "ready", "publish-active", "traffic"},
187		-	order)
	202	+	[]string{"create-new", "start-new", "ready", "publish-active", "traffic", "stop-old", "remove-old"},
	203	+	order, "the old container serves until the new one is published and routed; it is never withdrawn")
	204	+	assert.Empty(t, result.CleanupWarnings)
	205	+	}
	206	+
	207	+	// TestDeploy_OverlapRetiresOldGenerationWithItsOwnStopGrace proves the
	208	+	// superseded container is stopped with the grace of the generation being
	209	+	// stopped (stop_grace from ACTIVE), not the grace of its replacement.
	210	+	func TestDeploy_OverlapRetiresOldGenerationWithItsOwnStopGrace(t *testing.T) {
	211	+	ctx := context.Background()
	212	+	state, runtime, images, secrets := mockDeps(t)
	213	+	rev := mockRevision()
	214	+	rev.Spec.Services[0].StopGrace = 3 * time.Second
	215	+
	216	+	oldActive := replaceableActive()
	217	+	oldActive.Services["web"] = domain.AppEffectiveService{
	218	+	EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", Spec: graceSpec(45 * time.Second),
	219	+	}
	220	+	expectDeployPreflight(state, runtime, images, secrets, rev)
	221	+	expectNetworkProvision(runtime, "app-blog", 1)
	222	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(oldActive, true, nil)
	223	+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	224	+	expectSequentialCreate(state, runtime, nil, "c-new", 18080)
	225	+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
	226	+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
	227	+	expectRetirementMarker(state, "c-old")
	228	+	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
	229	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", 45*time.Second).Return(nil).Once()
	230	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
	231	+	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
	232	+
	233	+	svc := deployment.NewService(deployment.Deps{
	234	+	State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: &orderedTraffic{},
	235	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	236	+	func(context.Context, string, string) (int, error) { return 200, nil },
	237	+	func(context.Context, string) error { return nil },
	238	+	))
	239	+
	240	+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	241	+	require.NoError(t, err)
	242	+	}
	243	+
	244	+	// TestDeploy_OverlapOldRetirementFailureIsACleanupWarning proves a superseded
	245	+	// container that cannot be removed does not fail a deploy whose new generation
	246	+	// already serves: the leftover is a cleanup warning on the service and the
	247	+	// journal, and the step stays eligible for reconciliation.
	248	+	func TestDeploy_OverlapOldRetirementFailureIsACleanupWarning(t *testing.T) {
	249	+	ctx := context.Background()
	250	+	state, runtime, images, secrets := mockDeps(t)
	251	+	rev := mockRevision()
	252	+
	253	+	expectDeployPreflight(state, runtime, images, secrets, rev)
	254	+	expectNetworkProvision(runtime, "app-blog", 1)
	255	+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
	256	+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
	257	+	expectSequentialCreate(state, runtime, nil, "c-new", 18080)
	258	+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
	259	+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
	260	+	expectRetirementMarker(state, "c-old")
	261	+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
	262	+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
	263	+
	264	+	svc := deployment.NewService(deployment.Deps{
	265	+	State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: &orderedTraffic{},
	266	+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
	267	+	func(context.Context, string, string) (int, error) { return 200, nil },
	268	+	func(context.Context, string) error { return nil },
	269	+	))
	270	+
	271	+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
	272	+	require.NoError(t, err, "the new generation serves: a retirement leftover is not a deploy failure")
	273	+	assert.Equal(t, "deployed", result.Services["web"].Result)
	274	+	require.Len(t, result.CleanupWarnings, 1)
	275	+	assert.Equal(t, "c-old", result.CleanupWarnings[0].Leftover)
	276	+	assert.Equal(t, "web", result.CleanupWarnings[0].Service)
	277	+	assert.Contains(t, result.CleanupWarnings[0].Detail, "remove")
	278	+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-old")
	279	+	state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
188	280		}
189	281
190	282		// TestDeploy_PreflightFailureLeavesTheWorkloadUntouched proves preflight runs

@@ -219,15 +311,25 @@ func TestDeploy_PreflightFailureLeavesTheWorkloadUntouched(t *testing.T) {
219	311		state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
220	312		}
221	313
	314	+	// statefulMockRevision is mockRevision with a persistent volume, so the
	315	+	// service keeps the sequential replacement.
	316	+	func statefulMockRevision() domain.AppDesiredRevision {
	317	+	rev := mockRevision()
	318	+	rev.Spec.Services[0].Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}}
	319	+	return rev
	320	+	}
	321	+
222	322		// TestDeploy_WithdrawalFailureBlocksMutation proves a traffic withdrawal that
223		-	// cannot be applied leaves the running generation untouched: no container is
224		-	// stopped, removed, created, or started, and no ACTIVE state is published.
	323	+	// cannot be applied leaves the running generation of a single-writer service
	324	+	// untouched: no container is stopped, removed, created, or started, and no
	325	+	// ACTIVE state is published.
225	326		func TestDeploy_WithdrawalFailureBlocksMutation(t *testing.T) {
226	327		ctx := context.Background()
227	328		state, runtime, images, secrets := mockDeps(t)
228		-	rev := mockRevision()
	329	+	rev := statefulMockRevision()
229	330
230	331		expectDeployPreflight(state, runtime, images, secrets, rev)
	332	+	runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil)
231	333		state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
232	334		state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
233	335

@@ -304,7 +406,8 @@ func TestDeploy_FirstDeploymentRetiresNothing(t *testing.T) {
304	406		// TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate proves a
305	407		// replacement that is not ready is journaled as failed, that ACTIVE is never
306	408		// published for it, and that the unready candidate is removed. The superseded
307		-	// container is already gone and is never recreated.
	409	+	// container keeps serving: it is never withdrawn, stopped, removed, or
	410	+	// inhibited.
308	411		func TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate(t *testing.T) {
309	412		ctx := context.Background()
310	413		state, runtime, images, secrets := mockDeps(t)

@@ -317,7 +420,6 @@ func TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate(t *testing.T)
317	420		state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
318	421
319	422		var order []string
320		-	expectRetireOld(state, runtime, &order, "c-old")
321	423		expectSequentialCreate(state, runtime, &order, "c-new", 18080)
322	424		runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
323	425		runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).RunAndReturn(

@@ -350,10 +452,10 @@ func TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate(t *testing.T)
350	452		assert.NotContains(t, result.Services["web"].Error, "logs:", "raw logs must never be embedded in the public error")
351	453		assert.Equal(t, "c-new", result.Services["web"].After)
352	454
353		-	// ACTIVE was never repointed and the replaced generation's inhibition was
354		-	// never cleared: nothing claims the old workload is available again.
	455	+	// ACTIVE was never repointed and the old container was never touched.
355	456		state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
356	457		state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
	458	+	state.AssertNotCalled(t, "SaveRecoveryInhibition", mock.Anything, mock.Anything)
357	459
358	460		require.NotEmpty(t, saved)
359	461		last := saved[len(saved)-1]

@@ -362,9 +464,11 @@ func TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate(t *testing.T)
362	464		require.True(t, ok)
363	465		assert.Equal(t, domain.AppStepFailed, step.State)
364	466
365		-	assert.Equal(t, "remove-candidate", order[len(order)-1], "the unready candidate is the last thing removed")
366		-	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-new", mock.Anything)
367		-	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", true)
	467	+	assert.Equal(t, []string{"create-new", "start-new", "remove-candidate"}, order,
	468	+	"the old generation is never withdrawn: no withdraw or traffic event, the unready candidate is removed")
	469	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
	470	+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", mock.Anything)
	471	+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-old")
368	472		}
369	473
370	474		// TestDeploy_CandidateCleanupFailureStaysVisible proves a candidate that

@@ -381,7 +485,6 @@ func TestDeploy_CandidateCleanupFailureStaysVisible(t *testing.T) {
381	485		state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
382	486		state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
383	487
384		-	expectRetireOld(state, runtime, nil, "c-old")
385	488		expectSequentialCreate(state, runtime, nil, "c-new", 18080)
386	489		runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
387	490		runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(assert.AnError).Once()

@@ -424,7 +527,6 @@ func TestDeploy_CancellationDuringReadinessIsNotReportedAsSuccess(t *testing.T)
424	527		state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
425	528		state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
426	529
427		-	expectRetireOld(state, runtime, nil, "c-old")
428	530		expectSequentialCreate(state, runtime, nil, "c-new", 18080)
429	531		// The runtime refuses to remove the candidate with a canceled context.
430	532		runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(context.Canceled).Once()

@@ -547,6 +649,7 @@ func TestDeploy_MultiServiceStopsAtFirstFailure(t *testing.T) {
547	649		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
548	650		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-beta", Name: "beta"}, nil).Once()
549	651		runtime.EXPECT().StartContainer(mock.Anything, "c-beta").Return(nil).Once()
	652	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-beta").Return(&domain.Container{ID: "c-beta", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
550	653		runtime.EXPECT().GetContainerLogs(mock.Anything, "c-beta", false).Return(nil, assert.AnError).Once()
551	654		runtime.EXPECT().RemoveContainer(mock.Anything, "c-beta", true).Return(nil).Once()
552	655		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-beta").Return(nil).Once()

@@ -672,6 +775,7 @@ func TestDeploy_VolumeReplacementPreventsOverlappingWriters(t *testing.T) {
672	775		recordEvent(&order, "start-new")
673	776		return nil
674	777		}).Once()
	778	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
675	779		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
676	780		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
677	781		runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()

 			recordEvent(events, "start-new")
 			return nil
 		}).Once()
+	// Deploy observes the execution start right after the start.
+	runtime.EXPECT().InspectContainer(mock.Anything, containerID).Return(
+		&domain.Container{ID: containerID, Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, containerID, mock.Anything).Return(
 		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: hostPort, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
 }
+// expectRetirementMarker wires the durable retirement-pending marker an
+// overlap replacement records for the superseded container before the switch.
+func expectRetirementMarker(state *outmocks.MockAppState, containerID string) {
+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.MatchedBy(func(i domain.AppRecoveryInhibition) bool {
+		return i.ContainerID == containerID && i.Reason == domain.AppInhibitRetirementPending
+	})).Return(nil).Once()
+}
 // expectRetireOld wires the confirmed retirement of the superseded container.
 func expectRetireOld(
 	state *outmocks.MockAppState,
 	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", containerID).Return(nil).Once()
 }
-// TestDeploy_SequentialReplacementNeverOverlapsGenerations proves the
+// TestDeploy_StatelessReplacementOverlapsAndSwitchesBeforeRetiring proves the
-// replacement order for a public HTTP service without volumes: withdrawal is
+// replacement order for a public HTTP service without volumes, binds, or
-// confirmed first, the superseded container is stopped and removed before the
+// devices: the candidate is created and ready while the superseded container
-// replacement is created, readiness runs before ACTIVE is published, and the
+// keeps serving, ACTIVE and the proxy graph are switched to it, and only then
-// graph is applied last.
+// is the superseded container stopped and removed. The old container is never
-func TestDeploy_SequentialReplacementNeverOverlapsGenerations(t *testing.T) {
+// withdrawn from traffic and never inhibited.
+func TestDeploy_StatelessReplacementOverlapsAndSwitchesBeforeRetiring(t *testing.T) {
 	ctx := context.Background()
 	state, runtime, images, secrets := mockDeps(t)
 	rev := mockRevision()
 	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
 	var order []string
-	expectRetireOld(state, runtime, &order, "c-old")
 	expectSequentialCreate(state, runtime, &order, "c-new", 18080)
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
-	state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(
+	state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(active domain.AppActive) bool {
+		return active.Services["web"].Container == "c-new" && active.Services["web"].BackendBinds[8080] == 18080
+	})).RunAndReturn(
 		func(context.Context, domain.AppActive) error {
 			recordEvent(&order, "publish-active")
 			return nil
 		}).Once()
 	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
+	expectRetirementMarker(state, "c-old")
+	// Confirmed retirement drops the marker.
 	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
+	expectRetireOld(state, runtime, &order, "c-old")
 	var probedURL string
 	traffic := &orderedTraffic{events: &order}
 	assert.Equal(t, "deployed", result.Services["web"].Result)
 	assert.Contains(t, probedURL, "127.0.0.1:18080", "readiness dials the replacement's fresh loopback bind")
 	assert.Equal(t,
-		[]string{"withdraw", "stop-old", "remove-old", "create-new", "start-new", "ready", "publish-active", "traffic"},
+		[]string{"create-new", "start-new", "ready", "publish-active", "traffic", "stop-old", "remove-old"},
-		order)
+		order, "the old container serves until the new one is published and routed; it is never withdrawn")
+	assert.Empty(t, result.CleanupWarnings)
+}
+// TestDeploy_OverlapRetiresOldGenerationWithItsOwnStopGrace proves the
+// superseded container is stopped with the grace of the generation being
+// stopped (stop_grace from ACTIVE), not the grace of its replacement.
+func TestDeploy_OverlapRetiresOldGenerationWithItsOwnStopGrace(t *testing.T) {
+	ctx := context.Background()
+	state, runtime, images, secrets := mockDeps(t)
+	rev := mockRevision()
+	rev.Spec.Services[0].StopGrace = 3 * time.Second
+	oldActive := replaceableActive()
+	oldActive.Services["web"] = domain.AppEffectiveService{
+		EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", Spec: graceSpec(45 * time.Second),
+	}
+	expectDeployPreflight(state, runtime, images, secrets, rev)
+	expectNetworkProvision(runtime, "app-blog", 1)
+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(oldActive, true, nil)
+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
+	expectSequentialCreate(state, runtime, nil, "c-new", 18080)
+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
+	expectRetirementMarker(state, "c-old")
+	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", 45*time.Second).Return(nil).Once()
+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
+	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
+	svc := deployment.NewService(deployment.Deps{
+		State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: &orderedTraffic{},
+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
+		func(context.Context, string, string) (int, error) { return 200, nil },
+		func(context.Context, string) error { return nil },
+	))
+	_, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
+	require.NoError(t, err)
+}
+// TestDeploy_OverlapOldRetirementFailureIsACleanupWarning proves a superseded
+// container that cannot be removed does not fail a deploy whose new generation
+// already serves: the leftover is a cleanup warning on the service and the
+// journal, and the step stays eligible for reconciliation.
+func TestDeploy_OverlapOldRetirementFailureIsACleanupWarning(t *testing.T) {
+	ctx := context.Background()
+	state, runtime, images, secrets := mockDeps(t)
+	rev := mockRevision()
+	expectDeployPreflight(state, runtime, images, secrets, rev)
+	expectNetworkProvision(runtime, "app-blog", 1)
+	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
+	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
+	expectSequentialCreate(state, runtime, nil, "c-new", 18080)
+	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
+	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
+	expectRetirementMarker(state, "c-old")
+	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
+	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
+	svc := deployment.NewService(deployment.Deps{
+		State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: &orderedTraffic{},
+	}, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
+		func(context.Context, string, string) (int, error) { return 200, nil },
+		func(context.Context, string) error { return nil },
+	))
+	result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
+	require.NoError(t, err, "the new generation serves: a retirement leftover is not a deploy failure")
+	assert.Equal(t, "deployed", result.Services["web"].Result)
+	require.Len(t, result.CleanupWarnings, 1)
+	assert.Equal(t, "c-old", result.CleanupWarnings[0].Leftover)
+	assert.Equal(t, "web", result.CleanupWarnings[0].Service)
+	assert.Contains(t, result.CleanupWarnings[0].Detail, "remove")
+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-old")
+	state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
 }
 // TestDeploy_PreflightFailureLeavesTheWorkloadUntouched proves preflight runs
 	state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
 }
+// statefulMockRevision is mockRevision with a persistent volume, so the
+// service keeps the sequential replacement.
+func statefulMockRevision() domain.AppDesiredRevision {
+	rev := mockRevision()
+	rev.Spec.Services[0].Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}}
+	return rev
+}
 // TestDeploy_WithdrawalFailureBlocksMutation proves a traffic withdrawal that
-// cannot be applied leaves the running generation untouched: no container is
+// cannot be applied leaves the running generation of a single-writer service
-// stopped, removed, created, or started, and no ACTIVE state is published.
+// untouched: no container is stopped, removed, created, or started, and no
+// ACTIVE state is published.
 func TestDeploy_WithdrawalFailureBlocksMutation(t *testing.T) {
 	ctx := context.Background()
 	state, runtime, images, secrets := mockDeps(t)
-	rev := mockRevision()
+	rev := statefulMockRevision()
 	expectDeployPreflight(state, runtime, images, secrets, rev)
+	runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil)
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
 	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
 // TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate proves a
 // replacement that is not ready is journaled as failed, that ACTIVE is never
 // published for it, and that the unready candidate is removed. The superseded
-// container is already gone and is never recreated.
+// container keeps serving: it is never withdrawn, stopped, removed, or
+// inhibited.
 func TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate(t *testing.T) {
 	ctx := context.Background()
 	state, runtime, images, secrets := mockDeps(t)
 	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
 	var order []string
-	expectRetireOld(state, runtime, &order, "c-old")
 	expectSequentialCreate(state, runtime, &order, "c-new", 18080)
 	runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).RunAndReturn(
 	assert.NotContains(t, result.Services["web"].Error, "logs:", "raw logs must never be embedded in the public error")
 	assert.Equal(t, "c-new", result.Services["web"].After)
-	// ACTIVE was never repointed and the replaced generation's inhibition was
+	// ACTIVE was never repointed and the old container was never touched.
-	// never cleared: nothing claims the old workload is available again.
 	state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
 	state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
+	state.AssertNotCalled(t, "SaveRecoveryInhibition", mock.Anything, mock.Anything)
 	require.NotEmpty(t, saved)
 	last := saved[len(saved)-1]
 	require.True(t, ok)
 	assert.Equal(t, domain.AppStepFailed, step.State)
-	assert.Equal(t, "remove-candidate", order[len(order)-1], "the unready candidate is the last thing removed")
+	assert.Equal(t, []string{"create-new", "start-new", "remove-candidate"}, order,
-	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-new", mock.Anything)
+		"the old generation is never withdrawn: no withdraw or traffic event, the unready candidate is removed")
-	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", true)
+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", mock.Anything)
+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-old")
 }
 // TestDeploy_CandidateCleanupFailureStaysVisible proves a candidate that
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
 	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
-	expectRetireOld(state, runtime, nil, "c-old")
 	expectSequentialCreate(state, runtime, nil, "c-new", 18080)
 	runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(assert.AnError).Once()
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
 	state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
-	expectRetireOld(state, runtime, nil, "c-old")
 	expectSequentialCreate(state, runtime, nil, "c-new", 18080)
 	// The runtime refuses to remove the candidate with a canceled context.
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(context.Canceled).Once()
 	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-beta", Name: "beta"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-beta").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-beta").Return(&domain.Container{ID: "c-beta", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerLogs(mock.Anything, "c-beta", false).Return(nil, assert.AnError).Once()
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-beta", true).Return(nil).Once()
 	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-beta").Return(nil).Once()
 			recordEvent(&order, "start-new")
 			return nil
 		}).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
 		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()

M internal/usecase/deployment/service.go +8 -0


@@ -385,6 +385,14 @@ type ServiceResult struct {
385	385		After string
386	386		RestartUnsafe bool
387	387		Error string
	388	+	// VerifiedStartedAt is the execution start of After observed before its
	389	+	// readiness ran. The caller records it as a verified execution once the
	390	+	// generation is published and routed. Zero when unknown.
	391	+	VerifiedStartedAt time.Time
	392	+	// SupersededRunning is set by an overlap replacement: the container in
	393	+	// Before still runs beside After and carries a retirement-pending marker
	394	+	// until it is retired.
	395	+	SupersededRunning bool
388	396		// BackendBinds carries the created container's TCP loopback publishes
389	397		// (container port -> 127.0.0.1 host port) for the active record.
390	398		BackendBinds map[int]int


@@ -385,6 +385,14 @@ type ServiceResult struct {
385	After string	385		After string
386	RestartUnsafe bool	386		RestartUnsafe bool
387	Error string	387		Error string
		388	+	// VerifiedStartedAt is the execution start of After observed before its
		389	+	// readiness ran. The caller records it as a verified execution once the
		390	+	// generation is published and routed. Zero when unknown.
		391	+	VerifiedStartedAt time.Time
		392	+	// SupersededRunning is set by an overlap replacement: the container in
		393	+	// Before still runs beside After and carries a retirement-pending marker
		394	+	// until it is retired.
		395	+	SupersededRunning bool
388	// BackendBinds carries the created container's TCP loopback publishes	396		// BackendBinds carries the created container's TCP loopback publishes
389	// (container port -> 127.0.0.1 host port) for the active record.	397		// (container port -> 127.0.0.1 host port) for the active record.
390	BackendBinds map[int]int	398		BackendBinds map[int]int

M internal/usecase/deployment/service_test.go +2 -0


@@ -105,6 +105,7 @@ func TestReconcileBoot_ContinuesAfterAppFailure(t *testing.T) {
105	105		state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
106	106		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "bad").Return(nil, nil).Once()
107	107		runtime.EXPECT().IsContainerRunning(mock.Anything, "ctr-bad").Return(true, nil).Once()
	108	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-bad").Return(&domain.Container{ID: "ctr-bad", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
108	109		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "ctr-bad", mock.Anything).Return(nil, assert.AnError).Once()
109	110		// No traffic boundary is wired here, so the fail-closed bind
110	111		// withdrawal is a no-op and deployment writes no ACTIVE binds itself.

@@ -115,6 +116,7 @@ func TestReconcileBoot_ContinuesAfterAppFailure(t *testing.T) {
115	116		state.EXPECT().LoadLatestOperation(mock.Anything, "good").Return(domain.AppOperation{}, false, nil).Maybe()
116	117		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "good").Return(nil, nil).Once()
117	118		runtime.EXPECT().IsContainerRunning(mock.Anything, "ctr-good").Return(true, nil).Once()
	119	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-good").Return(&domain.Container{ID: "ctr-good", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
118	120		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "ctr-good", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32777, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
119	121		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
120	122		// Bind refresh persists the new bind (differs from recorded nil).


@@ -105,6 +105,7 @@ func TestReconcileBoot_ContinuesAfterAppFailure(t *testing.T) {
105	state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)	105		state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
106	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "bad").Return(nil, nil).Once()	106		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "bad").Return(nil, nil).Once()
107	runtime.EXPECT().IsContainerRunning(mock.Anything, "ctr-bad").Return(true, nil).Once()	107		runtime.EXPECT().IsContainerRunning(mock.Anything, "ctr-bad").Return(true, nil).Once()
		108	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-bad").Return(&domain.Container{ID: "ctr-bad", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
108	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "ctr-bad", mock.Anything).Return(nil, assert.AnError).Once()	109		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "ctr-bad", mock.Anything).Return(nil, assert.AnError).Once()
109	// No traffic boundary is wired here, so the fail-closed bind	110		// No traffic boundary is wired here, so the fail-closed bind
110	// withdrawal is a no-op and deployment writes no ACTIVE binds itself.	111		// withdrawal is a no-op and deployment writes no ACTIVE binds itself.

@@ -115,6 +116,7 @@ func TestReconcileBoot_ContinuesAfterAppFailure(t *testing.T) {
115	state.EXPECT().LoadLatestOperation(mock.Anything, "good").Return(domain.AppOperation{}, false, nil).Maybe()	116		state.EXPECT().LoadLatestOperation(mock.Anything, "good").Return(domain.AppOperation{}, false, nil).Maybe()
116	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "good").Return(nil, nil).Once()	117		state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "good").Return(nil, nil).Once()
117	runtime.EXPECT().IsContainerRunning(mock.Anything, "ctr-good").Return(true, nil).Once()	118		runtime.EXPECT().IsContainerRunning(mock.Anything, "ctr-good").Return(true, nil).Once()
		119	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-good").Return(&domain.Container{ID: "ctr-good", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
118	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "ctr-good", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32777, Protocol: domain.NetworkProtocolTCP}}, nil).Once()	120		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "ctr-good", mock.Anything).Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32777, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
119	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()	121		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
120	// Bind refresh persists the new bind (differs from recorded nil).	122		// Bind refresh persists the new bind (differs from recorded nil).

M internal/usecase/deployment/traffic_failure_test.go +22 -10


@@ -50,17 +50,19 @@ func TestDeploy_TrafficFailureRecordsFailedServiceStep(t *testing.T) {
50	50		state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil)
51	51
52	52		expectNetworkProvision(runtime, "app-blog", 1)
53		-	// The superseded generation is confirmed gone before the replacement is
54		-	// created: a deploy never overlaps two generations of one service.
55		-	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
56		-	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
57		-	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
	53	+	// The superseded container serves beside the candidate and must survive
	54	+	// the failed switch: no stop, no removal, no claim release.
58	55		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
59	56		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	57	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
60	58		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
61	59		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
62	60		state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
63		-	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
	61	+	// The durable retirement marker is the cleanup owner of the superseded
	62	+	// container; it is recorded before the switch and never cleared here.
	63	+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.MatchedBy(func(i domain.AppRecoveryInhibition) bool {
	64	+	return i.ContainerID == "c-old" && i.Reason == domain.AppInhibitRetirementPending
	65	+	})).Return(nil).Once()
64	66
65	67		var saved []domain.AppOperation
66	68		state.EXPECT().SaveOperation(mock.Anything, mock.Anything).RunAndReturn(

@@ -88,10 +90,19 @@ func TestDeploy_TrafficFailureRecordsFailedServiceStep(t *testing.T) {
88	90		step, ok := opStep(last, "service.web.replace")
89	91		require.True(t, ok)
90	92		assert.Equal(t, domain.AppStepFailed, step.State, "the service step must not claim success before traffic applied")
91		-	// The superseded container is gone: a rejected graph is reported as a
92		-	// failure of the new generation, never as a return to the old one.
93		-	runtime.AssertCalled(t, "RemoveContainer", mock.Anything, "c-old", false)
94		-	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-new", mock.Anything)
	93	+	// ACTIVE already names c-new while the proxy graph was refused, so the
	94	+	// superseded container may still be the one a stale graph reaches: it is
	95	+	// kept running. Both generations stay traceable in the journal for
	96	+	// reconciliation, and the leftover is reported.
	97	+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
	98	+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
	99	+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
	100	+	assert.Equal(t, "c-old", step.Before)
	101	+	assert.Equal(t, "c-new", step.After)
	102	+	require.Len(t, result.Services["web"].CleanupWarnings, 1)
	103	+	assert.Equal(t, "c-old", result.Services["web"].CleanupWarnings[0].Leftover)
	104	+	require.Len(t, last.Warnings, 1)
	105	+	assert.Equal(t, "c-old", last.Warnings[0].Leftover)
95	106		}
96	107
97	108		// TestStop_TrafficFailureRecordsTerminalFailure proves a rejected graph

@@ -107,6 +118,7 @@ func TestStop_TrafficFailureRecordsTerminalFailure(t *testing.T) {
107	118		state.EXPECT().Recover(mock.Anything).Return(nil)
108	119		state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
109	120		state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
	121	+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
110	122		runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(nil).Once()
111	123		runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
112	124		state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()

 	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil)
 	expectNetworkProvision(runtime, "app-blog", 1)
-	// The superseded generation is confirmed gone before the replacement is
+	// The superseded container serves beside the candidate and must survive
-	// created: a deploy never overlaps two generations of one service.
+	// the failed switch: no stop, no removal, no claim release.
-	runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
-	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
-	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
 	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
 		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
 	state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
-	state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
+	// The durable retirement marker is the cleanup owner of the superseded
+	// container; it is recorded before the switch and never cleared here.
+	state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.MatchedBy(func(i domain.AppRecoveryInhibition) bool {
+		return i.ContainerID == "c-old" && i.Reason == domain.AppInhibitRetirementPending
+	})).Return(nil).Once()
 	var saved []domain.AppOperation
 	state.EXPECT().SaveOperation(mock.Anything, mock.Anything).RunAndReturn(
 	step, ok := opStep(last, "service.web.replace")
 	require.True(t, ok)
 	assert.Equal(t, domain.AppStepFailed, step.State, "the service step must not claim success before traffic applied")
-	// The superseded container is gone: a rejected graph is reported as a
+	// ACTIVE already names c-new while the proxy graph was refused, so the
-	// failure of the new generation, never as a return to the old one.
+	// superseded container may still be the one a stale graph reaches: it is
-	runtime.AssertCalled(t, "RemoveContainer", mock.Anything, "c-old", false)
+	// kept running. Both generations stay traceable in the journal for
-	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-new", mock.Anything)
+	// reconciliation, and the leftover is reported.
+	runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
+	runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
+	state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
+	assert.Equal(t, "c-old", step.Before)
+	assert.Equal(t, "c-new", step.After)
+	require.Len(t, result.Services["web"].CleanupWarnings, 1)
+	assert.Equal(t, "c-old", result.Services["web"].CleanupWarnings[0].Leftover)
+	require.Len(t, last.Warnings, 1)
+	assert.Equal(t, "c-old", last.Warnings[0].Leftover)
 }
 // TestStop_TrafficFailureRecordsTerminalFailure proves a rejected graph
 	state.EXPECT().Recover(mock.Anything).Return(nil)
 	state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
 	state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil).Once()
+	state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
 	runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(nil).Once()
 	runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(nil).Once()
 	state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()

M internal/usecase/deployment/two_phase_deploy_test.go +2 -0


@@ -144,6 +144,7 @@ func expectHeadlessDeploy(
144	144		expectNetworkProvision(runtime, "app-blog", 1)
145	145		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil)
146	146		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil)
	147	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
147	148		state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil)
148	149		state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
149	150		return rev, saved

@@ -352,6 +353,7 @@ func TestStartDeploy_LiveClaimSurvivesForegroundReconciliation(t *testing.T) {
352	353		runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
353	354		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
354	355		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
	356	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
355	357		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
356	358		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
357	359


@@ -144,6 +144,7 @@ func expectHeadlessDeploy(
144	expectNetworkProvision(runtime, "app-blog", 1)	144		expectNetworkProvision(runtime, "app-blog", 1)
145	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil)	145		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil)
146	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil)	146		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil)
		147	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
147	state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil)	148		state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil)
148	state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)	149		state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
149	return rev, saved	150		return rev, saved

@@ -352,6 +353,7 @@ func TestStartDeploy_LiveClaimSurvivesForegroundReconciliation(t *testing.T) {
352	runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()	353		runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
353	runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()	354		runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
354	runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()	355		runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
		356	+	runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
355	runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(	357		runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
356	[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()	358		[]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
357		359