feat(daemon): preserve service across session purges
- commit
- 3e0d64d401f59ed7cfdb7047b441cde2eed9640f
- author
- brice · · 2026-09-17 07:18 UTC
- parent
- 0a1c4c5
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
36 files · +4505 -259
M AGENTS.md +3 -1
| @@ -77,7 +77,9 @@ Typed messages and negotiated version live in `internal/protocol`. Remote discov | |||
| 77 | 77 | ||
| 78 | 78 | - Ephemeral numbered sessions survive detach while the daemon retains them, but are not persisted. | |
| 79 | 79 | - Named sessions survive headless and persist across daemon restarts. | |
| 80 | - | - The daemon starts on first use and exits when the last session ends. | |
| 80 | + | - The daemon starts on first use and survives an empty session registry: final session removal is not shutdown, so it can be reused; it ends only on an explicit shutdown request or process cancellation. | |
| 81 | + | - A bulk purge (`kill --all`) takes a transient admission gate and bounds each phase with its own deadline: admission drain, stopped/broken sweep, and a shared live-teardown budget. A repository delete that ignores cancellation is detached behind an exact name/incarnation/created-time fence and reported as a typed failure instead of holding purge admission or control. | |
| 82 | + | - A live unit whose teardown aborts before destructive ownership stays registered, and its snapshot-coordinator quarantine is rolled back so the surviving session resumes checkpoint scheduling and its attached clients are unfrozen. | |
| 81 | 83 | - Each connection has a 15-second handshake budget from connect through the initial committed publication. | |
| 82 | 84 | - Local and remote attach use the same typed `Hello`/`Welcome` session protocol. `sessionwire` translates typed traffic to Protobuf envelopes carried by IPC, QUIC, or SSH stdio. | |
| 83 | 85 | - Remote attach bootstraps an authenticated direct QUIC connection by default; `VEV_REMOTE_TRANSPORT=stdio` explicitly selects an SSH-only carriage. | |
| @@ -77,7 +77,9 @@ Typed messages and negotiated version live in `internal/protocol`. Remote discov | |||||
| 77 | 77 | ||||
| 78 | - Ephemeral numbered sessions survive detach while the daemon retains them, but are not persisted. | 78 | - Ephemeral numbered sessions survive detach while the daemon retains them, but are not persisted. | ||
| 79 | - Named sessions survive headless and persist across daemon restarts. | 79 | - Named sessions survive headless and persist across daemon restarts. | ||
| 80 | - | - The daemon starts on first use and exits when the last session ends. | 80 | + | - The daemon starts on first use and survives an empty session registry: final session removal is not shutdown, so it can be reused; it ends only on an explicit shutdown request or process cancellation. |
| 81 | + | - A bulk purge (`kill --all`) takes a transient admission gate and bounds each phase with its own deadline: admission drain, stopped/broken sweep, and a shared live-teardown budget. A repository delete that ignores cancellation is detached behind an exact name/incarnation/created-time fence and reported as a typed failure instead of holding purge admission or control. | |||
| 82 | + | - A live unit whose teardown aborts before destructive ownership stays registered, and its snapshot-coordinator quarantine is rolled back so the surviving session resumes checkpoint scheduling and its attached clients are unfrozen. | |||
| 81 | - Each connection has a 15-second handshake budget from connect through the initial committed publication. | 83 | - Each connection has a 15-second handshake budget from connect through the initial committed publication. | ||
| 82 | - Local and remote attach use the same typed `Hello`/`Welcome` session protocol. `sessionwire` translates typed traffic to Protobuf envelopes carried by IPC, QUIC, or SSH stdio. | 84 | - Local and remote attach use the same typed `Hello`/`Welcome` session protocol. `sessionwire` translates typed traffic to Protobuf envelopes carried by IPC, QUIC, or SSH stdio. | ||
| 83 | - Remote attach bootstraps an authenticated direct QUIC connection by default; `VEV_REMOTE_TRANSPORT=stdio` explicitly selects an SSH-only carriage. | 85 | - Remote attach bootstraps an authenticated direct QUIC connection by default; `VEV_REMOTE_TRANSPORT=stdio` explicitly selects an SSH-only carriage. | ||
M README.md +1 -1
| @@ -42,7 +42,7 @@ vev --ui-observe ... expose passive observation for an attach | |||
| 42 | 42 | vev --ui-control ... expose observation and input control | |
| 43 | 43 | ``` | |
| 44 | 44 | ||
| 45 | - | The daemon starts on first use and exits after the last session. Numbered sessions survive detach; named sessions also survive daemon restarts. Exiting the final shell removes that session and returns to the most recently used session; with no previous session, vev exits. See [durable-session recovery](docs/durable-session-recovery.md). | |
| 45 | + | The daemon starts on first use and survives an empty session registry: removing the last session is not shutdown, so it can be reused. `kill --all` purges every session and leaves the daemon running; it ends only on an explicit `kill --daemon` or process cancellation. Numbered sessions survive detach; named sessions also survive daemon restarts. Exiting the final shell removes that session and returns to the most recently used session; with no previous session, vev exits. See [durable-session recovery](docs/durable-session-recovery.md). | |
| 46 | 46 | ||
| 47 | 47 | > [!NOTE] | |
| 48 | 48 | > A release that bumps vev's protocol version resets named sessions saved by an older protocol. Session names are retained, but layouts, tabs, terminal history, recovery transcripts, and process-recovery state are discarded. Each local or remote daemon applies this reset to its own sessions when it first starts with the new protocol. | |
| @@ -42,7 +42,7 @@ vev --ui-observe ... expose passive observation for an attach | |||||
| 42 | vev --ui-control ... expose observation and input control | 42 | vev --ui-control ... expose observation and input control | ||
| 43 | ``` | 43 | ``` | ||
| 44 | 44 | ||||
| 45 | - | The daemon starts on first use and exits after the last session. Numbered sessions survive detach; named sessions also survive daemon restarts. Exiting the final shell removes that session and returns to the most recently used session; with no previous session, vev exits. See [durable-session recovery](docs/durable-session-recovery.md). | 45 | + | The daemon starts on first use and survives an empty session registry: removing the last session is not shutdown, so it can be reused. `kill --all` purges every session and leaves the daemon running; it ends only on an explicit `kill --daemon` or process cancellation. Numbered sessions survive detach; named sessions also survive daemon restarts. Exiting the final shell removes that session and returns to the most recently used session; with no previous session, vev exits. See [durable-session recovery](docs/durable-session-recovery.md). |
| 46 | 46 | ||||
| 47 | > [!NOTE] | 47 | > [!NOTE] | ||
| 48 | > A release that bumps vev's protocol version resets named sessions saved by an older protocol. Session names are retained, but layouts, tabs, terminal history, recovery transcripts, and process-recovery state are discarded. Each local or remote daemon applies this reset to its own sessions when it first starts with the new protocol. | 48 | > A release that bumps vev's protocol version resets named sessions saved by an older protocol. Session names are retained, but layouts, tabs, terminal history, recovery transcripts, and process-recovery state are discarded. Each local or remote daemon applies this reset to its own sessions when it first starts with the new protocol. | ||
M docs/architecture.md +8 -0
| @@ -106,6 +106,14 @@ Attention is a daemon fact, not an overlay: a bell raised by a PTY, by a status- | |||
| 106 | 106 | ||
| 107 | 107 | Remote discovery has two owners with separated writers. The daemon composes its own store, catalogue client, cache, and monitor for the structured rows and exact-target validation its picker serves. The launching client composes a runner-scoped `remotes.HostRegistry` over its own store, catalogue client, and a runner-local catalogue cache that seeds from the durable snapshot and never writes it, so one monitor can never truncate the other's file. The registry projects that discovery through `ports.RemoteDirectory` and resolves each endpoint once into a `ports.RemoteEndpointBinding` — the typed dialer every session on that endpoint attaches through, plus the environment to advertise — which the runner reuses for its initial remote attach and for every later handoff. Transport mode, launch allowlist, and endpoint environment stay in `internal/app` behind `ports.RemoteEndpointFactory`; the client never inspects them, and a resolution failure is never cached, so a refused endpoint fails the handoff instead of falling back to another resolver. | |
| 108 | 108 | ||
| 109 | + | Current state: the daemon-side monitor is deliberately still active. `Serve` | |
| 110 | + | starts the runner through `daemon.WithRemoteMonitor` and joins it inside the | |
| 111 | + | bounded remote-shutdown budget, and the daemon keeps its own remote directory | |
| 112 | + | and host registry. That daemon-owned monitoring (and the host registry that | |
| 113 | + | feeds it) belongs to the coordinated P7 removal set; it is not removed, | |
| 114 | + | gated, or polled differently before that cutover, and no polling gate is | |
| 115 | + | introduced here (GO-001, deferred to P7). | |
| 116 | + | ||
| 109 | 117 | ## Session composition | |
| 110 | 118 | ||
| 111 | 119 | ```text | |
| @@ -106,6 +106,14 @@ Attention is a daemon fact, not an overlay: a bell raised by a PTY, by a status- | |||||
| 106 | 106 | ||||
| 107 | Remote discovery has two owners with separated writers. The daemon composes its own store, catalogue client, cache, and monitor for the structured rows and exact-target validation its picker serves. The launching client composes a runner-scoped `remotes.HostRegistry` over its own store, catalogue client, and a runner-local catalogue cache that seeds from the durable snapshot and never writes it, so one monitor can never truncate the other's file. The registry projects that discovery through `ports.RemoteDirectory` and resolves each endpoint once into a `ports.RemoteEndpointBinding` — the typed dialer every session on that endpoint attaches through, plus the environment to advertise — which the runner reuses for its initial remote attach and for every later handoff. Transport mode, launch allowlist, and endpoint environment stay in `internal/app` behind `ports.RemoteEndpointFactory`; the client never inspects them, and a resolution failure is never cached, so a refused endpoint fails the handoff instead of falling back to another resolver. | 107 | Remote discovery has two owners with separated writers. The daemon composes its own store, catalogue client, cache, and monitor for the structured rows and exact-target validation its picker serves. The launching client composes a runner-scoped `remotes.HostRegistry` over its own store, catalogue client, and a runner-local catalogue cache that seeds from the durable snapshot and never writes it, so one monitor can never truncate the other's file. The registry projects that discovery through `ports.RemoteDirectory` and resolves each endpoint once into a `ports.RemoteEndpointBinding` — the typed dialer every session on that endpoint attaches through, plus the environment to advertise — which the runner reuses for its initial remote attach and for every later handoff. Transport mode, launch allowlist, and endpoint environment stay in `internal/app` behind `ports.RemoteEndpointFactory`; the client never inspects them, and a resolution failure is never cached, so a refused endpoint fails the handoff instead of falling back to another resolver. | ||
| 108 | 108 | ||||
| 109 | + | Current state: the daemon-side monitor is deliberately still active. `Serve` | |||
| 110 | + | starts the runner through `daemon.WithRemoteMonitor` and joins it inside the | |||
| 111 | + | bounded remote-shutdown budget, and the daemon keeps its own remote directory | |||
| 112 | + | and host registry. That daemon-owned monitoring (and the host registry that | |||
| 113 | + | feeds it) belongs to the coordinated P7 removal set; it is not removed, | |||
| 114 | + | gated, or polled differently before that cutover, and no polling gate is | |||
| 115 | + | introduced here (GO-001, deferred to P7). | |||
| 116 | + | ||||
| 109 | ## Session composition | 117 | ## Session composition | ||
| 110 | 118 | ||||
| 111 | ```text | 119 | ```text | ||
M docs/configuration.md +3 -1
| @@ -179,7 +179,9 @@ Remote host commands, listing, and successful direct-attach learning are always | |||
| 179 | 179 | ||
| 180 | 180 | Set `VEV_LOG=debug`, `VEV_LOG=warn`, or `VEV_LOG=error` to change verbosity; the default is `info`. JSON-line logs such as `vev-daemon.log` live in `$XDG_STATE_HOME/vev`, or `~/.local/state/vev` when unset. The same state directory contains the strict session catalogue, any private pre-migration catalogue backup, notices, `hosts.json`, and `snapshots/`. The lifecycle lock and socket live in `$XDG_RUNTIME_DIR/vev` (with platform runtime fallbacks). | |
| 181 | 181 | ||
| 182 | - | Recovery events include `lifecycle_owner_wait`, `lifecycle_owner_acquired`, `lifecycle_owner_released`, `catalogue_validated`, `catalogue_compaction_recovery_complete`, `session_restore_complete`, `fallback_checkpoint_promoted`, `snapshot_head_repair_complete`, `session_degraded`, `snapshot_maintenance_progress`, `interrupted_transaction_recovery_complete`, and `daemon_startup_complete`. | |
| 182 | + | Recovery events include `lifecycle_owner_wait`, `lifecycle_owner_acquired`, `lifecycle_owner_released`, `catalogue_validated`, `catalogue_compaction_recovery_complete`, `session_restore_complete`, `fallback_checkpoint_promoted`, `snapshot_head_repair_complete`, `session_degraded`, `snapshot_garbage_collection_complete`, `interrupted_transaction_recovery_complete`, and `daemon_startup_complete`. | |
| 183 | + | ||
| 184 | + | Snapshot garbage collection is startup-only: the daemon runs one coordinator-owned pass before it publishes its socket, and no periodic snapshot collection runs afterward. The five-second idle maintenance loop compresses sealed scrollback pages and never collects snapshots or incarnations (GO-002, deferred). | |
| 183 | 185 | ||
| 184 | 186 | Catalogue failure is fail-closed: vev does not publish an empty replacement daemon. Preserve the state directory, inspect `catalogue_validation_failed`, correct storage or ownership problems, and retry without editing catalogue files. See [Durable session recovery](durable-session-recovery.md) for explicit recovery commands, migration, diagnostics, and the committed checkpoint plus up to two direct fallbacks retention policy. | |
| 185 | 187 | ||
| @@ -179,7 +179,9 @@ Remote host commands, listing, and successful direct-attach learning are always | |||||
| 179 | 179 | ||||
| 180 | Set `VEV_LOG=debug`, `VEV_LOG=warn`, or `VEV_LOG=error` to change verbosity; the default is `info`. JSON-line logs such as `vev-daemon.log` live in `$XDG_STATE_HOME/vev`, or `~/.local/state/vev` when unset. The same state directory contains the strict session catalogue, any private pre-migration catalogue backup, notices, `hosts.json`, and `snapshots/`. The lifecycle lock and socket live in `$XDG_RUNTIME_DIR/vev` (with platform runtime fallbacks). | 180 | Set `VEV_LOG=debug`, `VEV_LOG=warn`, or `VEV_LOG=error` to change verbosity; the default is `info`. JSON-line logs such as `vev-daemon.log` live in `$XDG_STATE_HOME/vev`, or `~/.local/state/vev` when unset. The same state directory contains the strict session catalogue, any private pre-migration catalogue backup, notices, `hosts.json`, and `snapshots/`. The lifecycle lock and socket live in `$XDG_RUNTIME_DIR/vev` (with platform runtime fallbacks). | ||
| 181 | 181 | ||||
| 182 | - | Recovery events include `lifecycle_owner_wait`, `lifecycle_owner_acquired`, `lifecycle_owner_released`, `catalogue_validated`, `catalogue_compaction_recovery_complete`, `session_restore_complete`, `fallback_checkpoint_promoted`, `snapshot_head_repair_complete`, `session_degraded`, `snapshot_maintenance_progress`, `interrupted_transaction_recovery_complete`, and `daemon_startup_complete`. | 182 | + | Recovery events include `lifecycle_owner_wait`, `lifecycle_owner_acquired`, `lifecycle_owner_released`, `catalogue_validated`, `catalogue_compaction_recovery_complete`, `session_restore_complete`, `fallback_checkpoint_promoted`, `snapshot_head_repair_complete`, `session_degraded`, `snapshot_garbage_collection_complete`, `interrupted_transaction_recovery_complete`, and `daemon_startup_complete`. |
| 183 | + | ||||
| 184 | + | Snapshot garbage collection is startup-only: the daemon runs one coordinator-owned pass before it publishes its socket, and no periodic snapshot collection runs afterward. The five-second idle maintenance loop compresses sealed scrollback pages and never collects snapshots or incarnations (GO-002, deferred). | |||
| 183 | 185 | ||||
| 184 | Catalogue failure is fail-closed: vev does not publish an empty replacement daemon. Preserve the state directory, inspect `catalogue_validation_failed`, correct storage or ownership problems, and retry without editing catalogue files. See [Durable session recovery](durable-session-recovery.md) for explicit recovery commands, migration, diagnostics, and the committed checkpoint plus up to two direct fallbacks retention policy. | 186 | Catalogue failure is fail-closed: vev does not publish an empty replacement daemon. Preserve the state directory, inspect `catalogue_validation_failed`, correct storage or ownership problems, and retry without editing catalogue files. See [Durable session recovery](durable-session-recovery.md) for explicit recovery commands, migration, diagnostics, and the committed checkpoint plus up to two direct fallbacks retention policy. | ||
| 185 | 187 | ||||
M docs/durable-session-recovery.md +4 -0
| @@ -6,6 +6,10 @@ vev opens named-session state only while holding `$XDG_RUNTIME_DIR/vev/lifecycle | |||
| 6 | 6 | ||
| 7 | 7 | `stopped` is safe fresh metadata, `restoring` is validation in progress, and `degraded` preserves uncertain state without starting a replacement shell. A degraded session remains visible in `vev ls`, but attach is refused until an explicit recovery action succeeds. A session reserved for durable purge is hidden from listings and pickers until deletion either completes or leaves a retained broken record. | |
| 8 | 8 | ||
| 9 | + | ## Bulk purge | |
| 10 | + | ||
| 11 | + | `kill --all` purges every live, stopped, and broken session while leaving the daemon running. Its admission gate is transient and each phase spends its own bounded deadline (admission drain, stopped/broken sweep, shared live teardown), so an uncooperative repository can never hold purge admission or control indefinitely. A durable delete that ignores cancellation is completed by a detached worker under the exact session name, incarnation, and creation time it captured, so it can never remove a same-name replacement; until it returns, the stopped authority still reserves the name and the purge reports a typed failure. A live unit whose teardown aborts before it owns destructive work stays registered, and its snapshot-coordinator quarantine is rolled back so it keeps checkpointing. | |
| 12 | + | ||
| 9 | 13 | ## Recovery commands | |
| 10 | 14 | ||
| 11 | 15 | - `vev cmd -s NAME session-recovery discard` | |
| @@ -6,6 +6,10 @@ vev opens named-session state only while holding `$XDG_RUNTIME_DIR/vev/lifecycle | |||||
| 6 | 6 | ||||
| 7 | `stopped` is safe fresh metadata, `restoring` is validation in progress, and `degraded` preserves uncertain state without starting a replacement shell. A degraded session remains visible in `vev ls`, but attach is refused until an explicit recovery action succeeds. A session reserved for durable purge is hidden from listings and pickers until deletion either completes or leaves a retained broken record. | 7 | `stopped` is safe fresh metadata, `restoring` is validation in progress, and `degraded` preserves uncertain state without starting a replacement shell. A degraded session remains visible in `vev ls`, but attach is refused until an explicit recovery action succeeds. A session reserved for durable purge is hidden from listings and pickers until deletion either completes or leaves a retained broken record. | ||
| 8 | 8 | ||||
| 9 | + | ## Bulk purge | |||
| 10 | + | ||||
| 11 | + | `kill --all` purges every live, stopped, and broken session while leaving the daemon running. Its admission gate is transient and each phase spends its own bounded deadline (admission drain, stopped/broken sweep, shared live teardown), so an uncooperative repository can never hold purge admission or control indefinitely. A durable delete that ignores cancellation is completed by a detached worker under the exact session name, incarnation, and creation time it captured, so it can never remove a same-name replacement; until it returns, the stopped authority still reserves the name and the purge reports a typed failure. A live unit whose teardown aborts before it owns destructive work stays registered, and its snapshot-coordinator quarantine is rolled back so it keeps checkpointing. | |||
| 12 | + | ||||
| 9 | ## Recovery commands | 13 | ## Recovery commands | ||
| 10 | 14 | ||||
| 11 | - `vev cmd -s NAME session-recovery discard` | 15 | - `vev cmd -s NAME session-recovery discard` | ||
M docs/protocol.md +9 -0
| @@ -188,6 +188,15 @@ while `pickerOpen`; only typed picker messages act. | |||
| 188 | 188 | and carry no async byte-budget charge; async sends are charged on | |
| 189 | 189 | admission and released exactly once on dequeue. | |
| 190 | 190 | ||
| 191 | + | Current state: a one-shot control client uses the daemon closing the | |
| 192 | + | connection (EOF) as its success signal, so a control failure whose error | |
| 193 | + | response cannot be delivered would otherwise be indistinguishable from | |
| 194 | + | success; the daemon only logs such a failed send | |
| 195 | + | (`daemon.logControlSendFailure`) before closing. The wire-level decision for that | |
| 196 | + | residual belongs to the P7 explicit result protocol, which replaces close-as- | |
| 197 | + | success with an explicit per-request result; until that cutover the close-based | |
| 198 | + | convention is kept unchanged (GO-005, deferred to P7). | |
| 199 | + | ||
| 191 | 200 | ## Limits | |
| 192 | 201 | ||
| 193 | 202 | | Item | Ceiling | Owner | | |
| @@ -188,6 +188,15 @@ while `pickerOpen`; only typed picker messages act. | |||||
| 188 | and carry no async byte-budget charge; async sends are charged on | 188 | and carry no async byte-budget charge; async sends are charged on | ||
| 189 | admission and released exactly once on dequeue. | 189 | admission and released exactly once on dequeue. | ||
| 190 | 190 | ||||
| 191 | + | Current state: a one-shot control client uses the daemon closing the | |||
| 192 | + | connection (EOF) as its success signal, so a control failure whose error | |||
| 193 | + | response cannot be delivered would otherwise be indistinguishable from | |||
| 194 | + | success; the daemon only logs such a failed send | |||
| 195 | + | (`daemon.logControlSendFailure`) before closing. The wire-level decision for that | |||
| 196 | + | residual belongs to the P7 explicit result protocol, which replaces close-as- | |||
| 197 | + | success with an explicit per-request result; until that cutover the close-based | |||
| 198 | + | convention is kept unchanged (GO-005, deferred to P7). | |||
| 199 | + | ||||
| 191 | ## Limits | 200 | ## Limits | ||
| 192 | 201 | ||||
| 193 | | Item | Ceiling | Owner | | 202 | | Item | Ceiling | Owner | | ||
M internal/app/daemon_integration_test.go +256 -26
| @@ -150,6 +150,34 @@ func killDaemon(dir string) error { | |||
| 150 | 150 | return requestKill(dir, protocol.KillDaemon) | |
| 151 | 151 | } | |
| 152 | 152 | ||
| 153 | + | // killNamedSession deletes one named live or stopped session over the control | |
| 154 | + | // connection, using the same wire scope the client sends for `kill -s NAME`. | |
| 155 | + | func killNamedSession(dir, name string) error { | |
| 156 | + | raw, err := ipc.DialContext(context.Background(), dir) | |
| 157 | + | if err != nil { | |
| 158 | + | return err | |
| 159 | + | } | |
| 160 | + | conn := sessionwire.NewClientConnection(raw) | |
| 161 | + | defer func() { _ = conn.Close() }() | |
| 162 | + | if err := conn.SendClient(protocol.Kill{Scope: protocol.KillSession, Name: name}); err != nil { | |
| 163 | + | return err | |
| 164 | + | } | |
| 165 | + | reply, err := receiveKillReply(conn, daemonStopTimeout) | |
| 166 | + | if errors.Is(err, context.DeadlineExceeded) { | |
| 167 | + | return err | |
| 168 | + | } | |
| 169 | + | if errors.Is(err, io.EOF) { | |
| 170 | + | return nil | |
| 171 | + | } | |
| 172 | + | if err != nil { | |
| 173 | + | return err | |
| 174 | + | } | |
| 175 | + | if em, ok := reply.(protocol.ErrorMsg); ok { | |
| 176 | + | return fmt.Errorf("kill failed: %s", em.Text) | |
| 177 | + | } | |
| 178 | + | return nil | |
| 179 | + | } | |
| 180 | + | ||
| 153 | 181 | func requestKill(dir string, scope protocol.KillScope) error { | |
| 154 | 182 | raw, err := ipc.DialContext(context.Background(), dir) | |
| 155 | 183 | if err != nil { | |
| @@ -333,6 +361,27 @@ func awaitText(t *testing.T, p *typedPump, sz domain.Size, want string) { | |||
| 333 | 361 | _ = awaitScreenText(t, p, sz, want) | |
| 334 | 362 | } | |
| 335 | 363 | ||
| 364 | + | // awaitDetached consumes typed messages until the daemon signals the session's | |
| 365 | + | // end with the expected Detached reason, ignoring interleaved Output frames. | |
| 366 | + | func awaitDetached(t *testing.T, p *typedPump, reason uint8) { | |
| 367 | + | t.Helper() | |
| 368 | + | timeout := time.After(5 * time.Second) | |
| 369 | + | for { | |
| 370 | + | select { | |
| 371 | + | case message, ok := <-p.ch: | |
| 372 | + | if !ok { | |
| 373 | + | t.Fatal("connection closed before Detached") | |
| 374 | + | } | |
| 375 | + | if detached, ok := message.(protocol.Detached); ok { | |
| 376 | + | require.Equal(t, reason, detached.Reason) | |
| 377 | + | return | |
| 378 | + | } | |
| 379 | + | case <-timeout: | |
| 380 | + | t.Fatal("timed out waiting for Detached") | |
| 381 | + | } | |
| 382 | + | } | |
| 383 | + | } | |
| 384 | + | ||
| 336 | 385 | // awaitScreenText is like awaitText, but returns the reconstructed screen text | |
| 337 | 386 | // at the point the wanted text appears so callers can make additional checks. | |
| 338 | 387 | func awaitScreenText(t *testing.T, p *typedPump, sz domain.Size, want string) string { | |
| @@ -597,12 +646,13 @@ func TestIntegration_EphemeralSurvivesDetachAndReattaches(t *testing.T) { | |||
| 597 | 646 | defer func() { _ = tr2.Close() }() | |
| 598 | 647 | awaitText(t, p2, sz, "MARKER") | |
| 599 | 648 | ||
| 600 | - | require.NoError(t, killAll(dir)) | |
| 649 | + | // An ephemeral session's daemon is ended only by the explicit daemon stop. | |
| 650 | + | require.NoError(t, killDaemon(dir)) | |
| 601 | 651 | select { | |
| 602 | 652 | case err := <-served: | |
| 603 | 653 | require.NoError(t, err) | |
| 604 | 654 | case <-time.After(5 * time.Second): | |
| 605 | - | t.Fatal("daemon did not shut down after kill all") | |
| 655 | + | t.Fatal("daemon did not stop after kill --daemon") | |
| 606 | 656 | } | |
| 607 | 657 | } | |
| 608 | 658 | ||
| @@ -679,12 +729,20 @@ func TestIntegration_KillDaemonPreservesMultipleNamedSessions(t *testing.T) { | |||
| 679 | 729 | {Name: "beta", State: protocol.SessionDown}, | |
| 680 | 730 | }, sessions.Sessions) | |
| 681 | 731 | ||
| 732 | + | // Kill-all purges every durable record but leaves this daemon serving. | |
| 682 | 733 | require.NoError(t, killAll(dir)) | |
| 734 | + | require.Empty(t, listRemoteSessions(t, dir).Sessions) | |
| 735 | + | select { | |
| 736 | + | case err := <-served: | |
| 737 | + | t.Fatalf("kill-all stopped the restarted daemon: %v", err) | |
| 738 | + | default: | |
| 739 | + | } | |
| 740 | + | require.NoError(t, killDaemon(dir)) | |
| 683 | 741 | select { | |
| 684 | 742 | case err := <-served: | |
| 685 | 743 | require.NoError(t, err) | |
| 686 | 744 | case <-time.After(5 * time.Second): | |
| 687 | - | t.Fatal("restarted daemon did not stop after kill all") | |
| 745 | + | t.Fatal("restarted daemon did not stop after kill --daemon") | |
| 688 | 746 | } | |
| 689 | 747 | ||
| 690 | 748 | served = start() | |
| @@ -721,6 +779,142 @@ func TestIntegration_NamedSurvivesReattach(t *testing.T) { | |||
| 721 | 779 | awaitText(t, p2, sz, "MARKER") | |
| 722 | 780 | } | |
| 723 | 781 | ||
| 782 | + | // TestIntegration_NamedSessionLastRemovalKeepsDaemonServing is the P4.1 | |
| 783 | + | // app-level contract (P4.1 review GO-003): a named session killed as the last | |
| 784 | + | // registry entry is removed without stopping Serve, and the same daemon process | |
| 785 | + | // accepts the same name again. It runs over the real socket, PTY, and | |
| 786 | + | // catalogue/recovery/snapshot persistence the production composition installs. | |
| 787 | + | // A named session's final live removal is a durable purge, so the re-attached | |
| 788 | + | // name carries a fresh persisted incarnation rather than resuming the deleted | |
| 789 | + | // one. | |
| 790 | + | func TestIntegration_NamedSessionLastRemovalKeepsDaemonServing(t *testing.T) { | |
| 791 | + | sz := domain.Size{Cols: 80, Rows: 24} | |
| 792 | + | stateDir := filepath.Join(t.TempDir(), "state") | |
| 793 | + | dir := filepath.Join(t.TempDir(), "runtime") | |
| 794 | + | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | |
| 795 | + | opened, err := persist.OpenOrCreate(stateDir) | |
| 796 | + | require.NoError(t, err) | |
| 797 | + | coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader) | |
| 798 | + | ||
| 799 | + | _, served := startDaemonInDir(t, dir, | |
| 800 | + | daemon.WithShell("/bin/cat", nil), | |
| 801 | + | daemon.WithCatalogue(opened.Catalogue, opened.Records), | |
| 802 | + | daemon.WithSnapshotRepository(repository), | |
| 803 | + | daemon.WithRecoveryCoordinator(coordinator), | |
| 804 | + | ) | |
| 805 | + | ||
| 806 | + | first, p1 := attach(t, dir, protocol.IntentNew, "work", sz) | |
| 807 | + | require.NoError(t, first.SendClient(protocol.Input{Data: []byte("ping\n")})) | |
| 808 | + | awaitText(t, p1, sz, "ping") | |
| 809 | + | firstRecord, ok, err := opened.Catalogue.Record("work") | |
| 810 | + | require.NoError(t, err) | |
| 811 | + | require.True(t, ok, "a live named session must own a durable record") | |
| 812 | + | ||
| 813 | + | // End it as the last session: the registry empties, but that is not daemon | |
| 814 | + | // shutdown in the P4.1 model. | |
| 815 | + | require.NoError(t, killNamedSession(dir, "work")) | |
| 816 | + | awaitDetached(t, p1, protocol.ReasonSessionKilled) | |
| 817 | + | require.Eventually(t, func() bool { return len(listRemoteSessions(t, dir).Sessions) == 0 }, | |
| 818 | + | 5*time.Second, 10*time.Millisecond, "session was not removed") | |
| 819 | + | select { | |
| 820 | + | case err := <-served: | |
| 821 | + | t.Fatalf("Serve stopped after the last session was removed: %v", err) | |
| 822 | + | default: | |
| 823 | + | } | |
| 824 | + | _, ok, err = opened.Catalogue.Record("work") | |
| 825 | + | require.NoError(t, err) | |
| 826 | + | require.False(t, ok, "final live removal purges the durable record") | |
| 827 | + | ||
| 828 | + | // The same daemon process accepts the same name again and can still serve a | |
| 829 | + | // fresh terminal operation. | |
| 830 | + | second, p2 := attach(t, dir, protocol.IntentNew, "work", sz) | |
| 831 | + | require.NoError(t, second.SendClient(protocol.Input{Data: []byte("pong\n")})) | |
| 832 | + | awaitText(t, p2, sz, "pong") | |
| 833 | + | sessions := listRemoteSessions(t, dir).Sessions | |
| 834 | + | require.Len(t, sessions, 1) | |
| 835 | + | require.Equal(t, "work", sessions[0].Name) | |
| 836 | + | require.Equal(t, protocol.SessionUp, sessions[0].State) | |
| 837 | + | secondRecord, ok, err := opened.Catalogue.Record("work") | |
| 838 | + | require.NoError(t, err) | |
| 839 | + | require.True(t, ok, "re-attached named session must persist its identity") | |
| 840 | + | require.NotEqual(t, firstRecord.IncarnationID, secondRecord.IncarnationID, | |
| 841 | + | "a re-created name must not reuse the purged incarnation") | |
| 842 | + | ||
| 843 | + | require.NoError(t, second.Close()) | |
| 844 | + | require.NoError(t, killDaemon(dir)) | |
| 845 | + | select { | |
| 846 | + | case err := <-served: | |
| 847 | + | require.NoError(t, err) | |
| 848 | + | case <-time.After(5 * time.Second): | |
| 849 | + | t.Fatal("daemon did not stop after kill --daemon") | |
| 850 | + | } | |
| 851 | + | } | |
| 852 | + | ||
| 853 | + | // TestIntegration_NamedSessionRestoresPersistedIdentity proves a stopped named | |
| 854 | + | // session is restored inside the daemon process that is already serving it, | |
| 855 | + | // keeping its exact persisted identity. The seed daemon stops while holding the | |
| 856 | + | // session as its only registry entry, which preserves it as stopped authority; | |
| 857 | + | // the daemon under test then resumes it over a real socket with the production | |
| 858 | + | // catalogue/recovery/snapshot options and serves a fresh operation. | |
| 859 | + | func TestIntegration_NamedSessionRestoresPersistedIdentity(t *testing.T) { | |
| 860 | + | sz := domain.Size{Cols: 80, Rows: 24} | |
| 861 | + | stateDir := filepath.Join(t.TempDir(), "state") | |
| 862 | + | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | |
| 863 | + | name := publishRestorableCheckpoint(t, stateDir, repository) | |
| 864 | + | ||
| 865 | + | opened, err := persist.OpenOrCreate(stateDir) | |
| 866 | + | require.NoError(t, err) | |
| 867 | + | stopped, ok, err := opened.Catalogue.Record(name) | |
| 868 | + | require.NoError(t, err) | |
| 869 | + | require.True(t, ok, "stopped named session must keep its durable record") | |
| 870 | + | require.NotNil(t, stopped.Committed, "stopped named session must keep its committed checkpoint") | |
| 871 | + | ||
| 872 | + | dir := filepath.Join(t.TempDir(), "runtime") | |
| 873 | + | coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader) | |
| 874 | + | _, served := startDaemonInDir(t, dir, | |
| 875 | + | daemon.WithShell("/bin/cat", nil), | |
| 876 | + | daemon.WithCatalogue(opened.Catalogue, opened.Records), | |
| 877 | + | daemon.WithSnapshotRepository(repository), | |
| 878 | + | daemon.WithRecoveryCoordinator(coordinator), | |
| 879 | + | ) | |
| 880 | + | require.Equal(t, []protocol.SessionInfo{{Name: name, State: protocol.SessionDown}}, listRemoteSessions(t, dir).Sessions) | |
| 881 | + | select { | |
| 882 | + | case err := <-served: | |
| 883 | + | t.Fatalf("Serve stopped before the restore: %v", err) | |
| 884 | + | default: | |
| 885 | + | } | |
| 886 | + | ||
| 887 | + | conn, p := attach(t, dir, protocol.IntentAttach, name, sz) | |
| 888 | + | require.NoError(t, conn.SendClient(protocol.Input{Data: []byte("revived\n")})) | |
| 889 | + | awaitText(t, p, sz, "revived") | |
| 890 | + | require.Eventually(t, func() bool { | |
| 891 | + | sessions := listRemoteSessions(t, dir).Sessions | |
| 892 | + | return len(sessions) == 1 && sessions[0].State == protocol.SessionUp | |
| 893 | + | }, 5*time.Second, 10*time.Millisecond, "named session was not restored") | |
| 894 | + | ||
| 895 | + | restored, ok, err := opened.Catalogue.Record(name) | |
| 896 | + | require.NoError(t, err) | |
| 897 | + | require.True(t, ok) | |
| 898 | + | require.Equal(t, stopped.IncarnationID, restored.IncarnationID, | |
| 899 | + | "restore must retain the persisted incarnation") | |
| 900 | + | require.Equal(t, stopped.CreatedAt, restored.CreatedAt, | |
| 901 | + | "restore must retain the persisted lifecycle timestamp") | |
| 902 | + | ||
| 903 | + | require.NoError(t, conn.Close()) | |
| 904 | + | select { | |
| 905 | + | case err := <-served: | |
| 906 | + | t.Fatalf("Serve stopped while the restored session was live: %v", err) | |
| 907 | + | default: | |
| 908 | + | } | |
| 909 | + | require.NoError(t, killDaemon(dir)) | |
| 910 | + | select { | |
| 911 | + | case err := <-served: | |
| 912 | + | require.NoError(t, err) | |
| 913 | + | case <-time.After(5 * time.Second): | |
| 914 | + | t.Fatal("daemon did not stop after kill --daemon") | |
| 915 | + | } | |
| 916 | + | } | |
| 917 | + | ||
| 724 | 918 | func TestMultipleClientsOneLifecycleOwner(t *testing.T) { | |
| 725 | 919 | dir := filepath.Join(t.TempDir(), "runtime") | |
| 726 | 920 | owner, err := lifecycle.TryAcquire(dir) | |
| @@ -1495,31 +1689,67 @@ func (*integrationTransport) Close() error { return nil } | |||
| 1495 | 1689 | func (*integrationTransport) LocalAddr() net.Addr { return nil } | |
| 1496 | 1690 | func (*integrationTransport) RemoteAddr() net.Addr { return nil } | |
| 1497 | 1691 | ||
| 1498 | - | func TestIntegration_KillAllShutsDownDaemon(t *testing.T) { | |
| 1692 | + | // TestIntegration_KillAllPurgesSessionsAndKeepsDaemon is the P4.2 app-level | |
| 1693 | + | // contract: `kill --all` removes every live and stopped session but leaves the | |
| 1694 | + | // daemon serving, and the same process accepts fresh named and ephemeral | |
| 1695 | + | // sessions that start PTYs, forward input, and open a tab through the ordinary | |
| 1696 | + | // command palette. Only the distinct explicit daemon stop ends it. | |
| 1697 | + | func TestIntegration_KillAllPurgesSessionsAndKeepsDaemon(t *testing.T) { | |
| 1499 | 1698 | sz := domain.Size{Cols: 80, Rows: 24} | |
| 1500 | - | dir, served := startDaemon(t, daemon.WithShell("/bin/sh", []string{"-c", "sleep 30"})) | |
| 1501 | - | ||
| 1502 | - | tr1, _ := attach(t, dir, protocol.IntentNew, "one", sz) | |
| 1503 | - | require.NoError(t, tr1.Close()) | |
| 1504 | - | tr2, _ := attach(t, dir, protocol.IntentNew, "two", sz) | |
| 1505 | - | require.NoError(t, tr2.Close()) | |
| 1506 | - | ||
| 1507 | - | killRaw, err := ipc.DialContext(context.Background(), dir) | |
| 1508 | - | require.NoError(t, err) | |
| 1509 | - | killTr := sessionwire.NewClientConnection(killRaw) | |
| 1510 | - | defer func() { _ = killTr.Close() }() | |
| 1511 | - | require.NoError(t, killTr.SendClient(protocol.Kill{Scope: protocol.KillAll})) | |
| 1699 | + | dir, served := startDaemon(t, daemon.WithShell("/bin/cat", nil)) | |
| 1700 | + | ||
| 1701 | + | first, firstPump := attach(t, dir, protocol.IntentNew, "one", sz) | |
| 1702 | + | require.NoError(t, first.SendClient(protocol.Input{Data: []byte("alpha\n")})) | |
| 1703 | + | awaitText(t, firstPump, sz, "alpha") | |
| 1704 | + | // A detached named session becomes stopped authority, so kill-all must purge | |
| 1705 | + | // both the live and the stopped record in one operation. | |
| 1706 | + | second, _ := attach(t, dir, protocol.IntentNew, "two", sz) | |
| 1707 | + | require.NoError(t, second.Close()) | |
| 1708 | + | require.Len(t, listRemoteSessions(t, dir).Sessions, 2) | |
| 1512 | 1709 | ||
| 1513 | - | _, err = killTr.ReceiveServer() | |
| 1514 | - | require.ErrorIs(t, err, io.EOF) | |
| 1515 | - | var serveErr error | |
| 1710 | + | require.NoError(t, killAll(dir)) | |
| 1711 | + | // The daemon survives the purge: Serve has not returned and the registry is | |
| 1712 | + | // empty. | |
| 1713 | + | select { | |
| 1714 | + | case err := <-served: | |
| 1715 | + | t.Fatalf("KillAll stopped Serve: %v", err) | |
| 1716 | + | default: | |
| 1717 | + | } | |
| 1718 | + | require.Eventually(t, func() bool { return len(listRemoteSessions(t, dir).Sessions) == 0 }, | |
| 1719 | + | 5*time.Second, 10*time.Millisecond, "kill-all did not purge every session") | |
| 1720 | + | ||
| 1721 | + | // Reuse: a fresh named session starts its PTY and forwards input. | |
| 1722 | + | third, thirdPump := attach(t, dir, protocol.IntentNew, "again", sz) | |
| 1723 | + | require.NoError(t, third.SendClient(protocol.Input{Data: []byte("beta\n")})) | |
| 1724 | + | awaitText(t, thirdPump, sz, "beta") | |
| 1725 | + | // A fresh ephemeral session is admitted too. | |
| 1726 | + | fourth, fourthPump := attach(t, dir, protocol.IntentEphemeral, "", sz) | |
| 1727 | + | require.NoError(t, fourth.SendClient(protocol.Input{Data: []byte("gamma\n")})) | |
| 1728 | + | awaitText(t, fourthPump, sz, "gamma") | |
| 1729 | + | ||
| 1730 | + | // A tab opened after the purge is real, live topology. | |
| 1731 | + | require.NoError(t, third.SendClient(protocol.Input{Data: []byte("\x1b ")})) | |
| 1732 | + | awaitText(t, thirdPump, sz, "Commands") | |
| 1733 | + | require.NoError(t, third.SendClient(protocol.Input{Data: []byte("CNT\r")})) | |
| 1516 | 1734 | require.Eventually(t, func() bool { | |
| 1517 | - | select { | |
| 1518 | - | case serveErr = <-served: | |
| 1519 | - | return true | |
| 1520 | - | default: | |
| 1521 | - | return false | |
| 1735 | + | for _, info := range listRemoteSessions(t, dir).Sessions { | |
| 1736 | + | if info.Name == "again" { | |
| 1737 | + | return info.Tabs == 2 | |
| 1738 | + | } | |
| 1522 | 1739 | } | |
| 1523 | - | }, 15*time.Second, 10*time.Millisecond, "daemon did not shut down after kill all") | |
| 1524 | - | require.NoError(t, serveErr) | |
| 1740 | + | return false | |
| 1741 | + | }, 5*time.Second, 10*time.Millisecond, "a tab created after the purge was not published") | |
| 1742 | + | ||
| 1743 | + | _ = first.Close() | |
| 1744 | + | require.NoError(t, third.Close()) | |
| 1745 | + | require.NoError(t, fourth.Close()) | |
| 1746 | + | ||
| 1747 | + | // Only the distinct explicit daemon stop ends the daemon. | |
| 1748 | + | require.NoError(t, killDaemon(dir)) | |
| 1749 | + | select { | |
| 1750 | + | case err := <-served: | |
| 1751 | + | require.NoError(t, err) | |
| 1752 | + | case <-time.After(5 * time.Second): | |
| 1753 | + | t.Fatal("daemon did not stop after kill --daemon") | |
| 1754 | + | } | |
| 1525 | 1755 | } | |
| @@ -150,6 +150,34 @@ func killDaemon(dir string) error { | |||||
| 150 | return requestKill(dir, protocol.KillDaemon) | 150 | return requestKill(dir, protocol.KillDaemon) | ||
| 151 | } | 151 | } | ||
| 152 | 152 | ||||
| 153 | + | // killNamedSession deletes one named live or stopped session over the control | |||
| 154 | + | // connection, using the same wire scope the client sends for `kill -s NAME`. | |||
| 155 | + | func killNamedSession(dir, name string) error { | |||
| 156 | + | raw, err := ipc.DialContext(context.Background(), dir) | |||
| 157 | + | if err != nil { | |||
| 158 | + | return err | |||
| 159 | + | } | |||
| 160 | + | conn := sessionwire.NewClientConnection(raw) | |||
| 161 | + | defer func() { _ = conn.Close() }() | |||
| 162 | + | if err := conn.SendClient(protocol.Kill{Scope: protocol.KillSession, Name: name}); err != nil { | |||
| 163 | + | return err | |||
| 164 | + | } | |||
| 165 | + | reply, err := receiveKillReply(conn, daemonStopTimeout) | |||
| 166 | + | if errors.Is(err, context.DeadlineExceeded) { | |||
| 167 | + | return err | |||
| 168 | + | } | |||
| 169 | + | if errors.Is(err, io.EOF) { | |||
| 170 | + | return nil | |||
| 171 | + | } | |||
| 172 | + | if err != nil { | |||
| 173 | + | return err | |||
| 174 | + | } | |||
| 175 | + | if em, ok := reply.(protocol.ErrorMsg); ok { | |||
| 176 | + | return fmt.Errorf("kill failed: %s", em.Text) | |||
| 177 | + | } | |||
| 178 | + | return nil | |||
| 179 | + | } | |||
| 180 | + | ||||
| 153 | func requestKill(dir string, scope protocol.KillScope) error { | 181 | func requestKill(dir string, scope protocol.KillScope) error { | ||
| 154 | raw, err := ipc.DialContext(context.Background(), dir) | 182 | raw, err := ipc.DialContext(context.Background(), dir) | ||
| 155 | if err != nil { | 183 | if err != nil { | ||
| @@ -333,6 +361,27 @@ func awaitText(t *testing.T, p *typedPump, sz domain.Size, want string) { | |||||
| 333 | _ = awaitScreenText(t, p, sz, want) | 361 | _ = awaitScreenText(t, p, sz, want) | ||
| 334 | } | 362 | } | ||
| 335 | 363 | ||||
| 364 | + | // awaitDetached consumes typed messages until the daemon signals the session's | |||
| 365 | + | // end with the expected Detached reason, ignoring interleaved Output frames. | |||
| 366 | + | func awaitDetached(t *testing.T, p *typedPump, reason uint8) { | |||
| 367 | + | t.Helper() | |||
| 368 | + | timeout := time.After(5 * time.Second) | |||
| 369 | + | for { | |||
| 370 | + | select { | |||
| 371 | + | case message, ok := <-p.ch: | |||
| 372 | + | if !ok { | |||
| 373 | + | t.Fatal("connection closed before Detached") | |||
| 374 | + | } | |||
| 375 | + | if detached, ok := message.(protocol.Detached); ok { | |||
| 376 | + | require.Equal(t, reason, detached.Reason) | |||
| 377 | + | return | |||
| 378 | + | } | |||
| 379 | + | case <-timeout: | |||
| 380 | + | t.Fatal("timed out waiting for Detached") | |||
| 381 | + | } | |||
| 382 | + | } | |||
| 383 | + | } | |||
| 384 | + | ||||
| 336 | // awaitScreenText is like awaitText, but returns the reconstructed screen text | 385 | // awaitScreenText is like awaitText, but returns the reconstructed screen text | ||
| 337 | // at the point the wanted text appears so callers can make additional checks. | 386 | // at the point the wanted text appears so callers can make additional checks. | ||
| 338 | func awaitScreenText(t *testing.T, p *typedPump, sz domain.Size, want string) string { | 387 | func awaitScreenText(t *testing.T, p *typedPump, sz domain.Size, want string) string { | ||
| @@ -597,12 +646,13 @@ func TestIntegration_EphemeralSurvivesDetachAndReattaches(t *testing.T) { | |||||
| 597 | defer func() { _ = tr2.Close() }() | 646 | defer func() { _ = tr2.Close() }() | ||
| 598 | awaitText(t, p2, sz, "MARKER") | 647 | awaitText(t, p2, sz, "MARKER") | ||
| 599 | 648 | ||||
| 600 | - | require.NoError(t, killAll(dir)) | 649 | + | // An ephemeral session's daemon is ended only by the explicit daemon stop. |
| 650 | + | require.NoError(t, killDaemon(dir)) | |||
| 601 | select { | 651 | select { | ||
| 602 | case err := <-served: | 652 | case err := <-served: | ||
| 603 | require.NoError(t, err) | 653 | require.NoError(t, err) | ||
| 604 | case <-time.After(5 * time.Second): | 654 | case <-time.After(5 * time.Second): | ||
| 605 | - | t.Fatal("daemon did not shut down after kill all") | 655 | + | t.Fatal("daemon did not stop after kill --daemon") |
| 606 | } | 656 | } | ||
| 607 | } | 657 | } | ||
| 608 | 658 | ||||
| @@ -679,12 +729,20 @@ func TestIntegration_KillDaemonPreservesMultipleNamedSessions(t *testing.T) { | |||||
| 679 | {Name: "beta", State: protocol.SessionDown}, | 729 | {Name: "beta", State: protocol.SessionDown}, | ||
| 680 | }, sessions.Sessions) | 730 | }, sessions.Sessions) | ||
| 681 | 731 | ||||
| 732 | + | // Kill-all purges every durable record but leaves this daemon serving. | |||
| 682 | require.NoError(t, killAll(dir)) | 733 | require.NoError(t, killAll(dir)) | ||
| 734 | + | require.Empty(t, listRemoteSessions(t, dir).Sessions) | |||
| 735 | + | select { | |||
| 736 | + | case err := <-served: | |||
| 737 | + | t.Fatalf("kill-all stopped the restarted daemon: %v", err) | |||
| 738 | + | default: | |||
| 739 | + | } | |||
| 740 | + | require.NoError(t, killDaemon(dir)) | |||
| 683 | select { | 741 | select { | ||
| 684 | case err := <-served: | 742 | case err := <-served: | ||
| 685 | require.NoError(t, err) | 743 | require.NoError(t, err) | ||
| 686 | case <-time.After(5 * time.Second): | 744 | case <-time.After(5 * time.Second): | ||
| 687 | - | t.Fatal("restarted daemon did not stop after kill all") | 745 | + | t.Fatal("restarted daemon did not stop after kill --daemon") |
| 688 | } | 746 | } | ||
| 689 | 747 | ||||
| 690 | served = start() | 748 | served = start() | ||
| @@ -721,6 +779,142 @@ func TestIntegration_NamedSurvivesReattach(t *testing.T) { | |||||
| 721 | awaitText(t, p2, sz, "MARKER") | 779 | awaitText(t, p2, sz, "MARKER") | ||
| 722 | } | 780 | } | ||
| 723 | 781 | ||||
| 782 | + | // TestIntegration_NamedSessionLastRemovalKeepsDaemonServing is the P4.1 | |||
| 783 | + | // app-level contract (P4.1 review GO-003): a named session killed as the last | |||
| 784 | + | // registry entry is removed without stopping Serve, and the same daemon process | |||
| 785 | + | // accepts the same name again. It runs over the real socket, PTY, and | |||
| 786 | + | // catalogue/recovery/snapshot persistence the production composition installs. | |||
| 787 | + | // A named session's final live removal is a durable purge, so the re-attached | |||
| 788 | + | // name carries a fresh persisted incarnation rather than resuming the deleted | |||
| 789 | + | // one. | |||
| 790 | + | func TestIntegration_NamedSessionLastRemovalKeepsDaemonServing(t *testing.T) { | |||
| 791 | + | sz := domain.Size{Cols: 80, Rows: 24} | |||
| 792 | + | stateDir := filepath.Join(t.TempDir(), "state") | |||
| 793 | + | dir := filepath.Join(t.TempDir(), "runtime") | |||
| 794 | + | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | |||
| 795 | + | opened, err := persist.OpenOrCreate(stateDir) | |||
| 796 | + | require.NoError(t, err) | |||
| 797 | + | coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader) | |||
| 798 | + | ||||
| 799 | + | _, served := startDaemonInDir(t, dir, | |||
| 800 | + | daemon.WithShell("/bin/cat", nil), | |||
| 801 | + | daemon.WithCatalogue(opened.Catalogue, opened.Records), | |||
| 802 | + | daemon.WithSnapshotRepository(repository), | |||
| 803 | + | daemon.WithRecoveryCoordinator(coordinator), | |||
| 804 | + | ) | |||
| 805 | + | ||||
| 806 | + | first, p1 := attach(t, dir, protocol.IntentNew, "work", sz) | |||
| 807 | + | require.NoError(t, first.SendClient(protocol.Input{Data: []byte("ping\n")})) | |||
| 808 | + | awaitText(t, p1, sz, "ping") | |||
| 809 | + | firstRecord, ok, err := opened.Catalogue.Record("work") | |||
| 810 | + | require.NoError(t, err) | |||
| 811 | + | require.True(t, ok, "a live named session must own a durable record") | |||
| 812 | + | ||||
| 813 | + | // End it as the last session: the registry empties, but that is not daemon | |||
| 814 | + | // shutdown in the P4.1 model. | |||
| 815 | + | require.NoError(t, killNamedSession(dir, "work")) | |||
| 816 | + | awaitDetached(t, p1, protocol.ReasonSessionKilled) | |||
| 817 | + | require.Eventually(t, func() bool { return len(listRemoteSessions(t, dir).Sessions) == 0 }, | |||
| 818 | + | 5*time.Second, 10*time.Millisecond, "session was not removed") | |||
| 819 | + | select { | |||
| 820 | + | case err := <-served: | |||
| 821 | + | t.Fatalf("Serve stopped after the last session was removed: %v", err) | |||
| 822 | + | default: | |||
| 823 | + | } | |||
| 824 | + | _, ok, err = opened.Catalogue.Record("work") | |||
| 825 | + | require.NoError(t, err) | |||
| 826 | + | require.False(t, ok, "final live removal purges the durable record") | |||
| 827 | + | ||||
| 828 | + | // The same daemon process accepts the same name again and can still serve a | |||
| 829 | + | // fresh terminal operation. | |||
| 830 | + | second, p2 := attach(t, dir, protocol.IntentNew, "work", sz) | |||
| 831 | + | require.NoError(t, second.SendClient(protocol.Input{Data: []byte("pong\n")})) | |||
| 832 | + | awaitText(t, p2, sz, "pong") | |||
| 833 | + | sessions := listRemoteSessions(t, dir).Sessions | |||
| 834 | + | require.Len(t, sessions, 1) | |||
| 835 | + | require.Equal(t, "work", sessions[0].Name) | |||
| 836 | + | require.Equal(t, protocol.SessionUp, sessions[0].State) | |||
| 837 | + | secondRecord, ok, err := opened.Catalogue.Record("work") | |||
| 838 | + | require.NoError(t, err) | |||
| 839 | + | require.True(t, ok, "re-attached named session must persist its identity") | |||
| 840 | + | require.NotEqual(t, firstRecord.IncarnationID, secondRecord.IncarnationID, | |||
| 841 | + | "a re-created name must not reuse the purged incarnation") | |||
| 842 | + | ||||
| 843 | + | require.NoError(t, second.Close()) | |||
| 844 | + | require.NoError(t, killDaemon(dir)) | |||
| 845 | + | select { | |||
| 846 | + | case err := <-served: | |||
| 847 | + | require.NoError(t, err) | |||
| 848 | + | case <-time.After(5 * time.Second): | |||
| 849 | + | t.Fatal("daemon did not stop after kill --daemon") | |||
| 850 | + | } | |||
| 851 | + | } | |||
| 852 | + | ||||
| 853 | + | // TestIntegration_NamedSessionRestoresPersistedIdentity proves a stopped named | |||
| 854 | + | // session is restored inside the daemon process that is already serving it, | |||
| 855 | + | // keeping its exact persisted identity. The seed daemon stops while holding the | |||
| 856 | + | // session as its only registry entry, which preserves it as stopped authority; | |||
| 857 | + | // the daemon under test then resumes it over a real socket with the production | |||
| 858 | + | // catalogue/recovery/snapshot options and serves a fresh operation. | |||
| 859 | + | func TestIntegration_NamedSessionRestoresPersistedIdentity(t *testing.T) { | |||
| 860 | + | sz := domain.Size{Cols: 80, Rows: 24} | |||
| 861 | + | stateDir := filepath.Join(t.TempDir(), "state") | |||
| 862 | + | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | |||
| 863 | + | name := publishRestorableCheckpoint(t, stateDir, repository) | |||
| 864 | + | ||||
| 865 | + | opened, err := persist.OpenOrCreate(stateDir) | |||
| 866 | + | require.NoError(t, err) | |||
| 867 | + | stopped, ok, err := opened.Catalogue.Record(name) | |||
| 868 | + | require.NoError(t, err) | |||
| 869 | + | require.True(t, ok, "stopped named session must keep its durable record") | |||
| 870 | + | require.NotNil(t, stopped.Committed, "stopped named session must keep its committed checkpoint") | |||
| 871 | + | ||||
| 872 | + | dir := filepath.Join(t.TempDir(), "runtime") | |||
| 873 | + | coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader) | |||
| 874 | + | _, served := startDaemonInDir(t, dir, | |||
| 875 | + | daemon.WithShell("/bin/cat", nil), | |||
| 876 | + | daemon.WithCatalogue(opened.Catalogue, opened.Records), | |||
| 877 | + | daemon.WithSnapshotRepository(repository), | |||
| 878 | + | daemon.WithRecoveryCoordinator(coordinator), | |||
| 879 | + | ) | |||
| 880 | + | require.Equal(t, []protocol.SessionInfo{{Name: name, State: protocol.SessionDown}}, listRemoteSessions(t, dir).Sessions) | |||
| 881 | + | select { | |||
| 882 | + | case err := <-served: | |||
| 883 | + | t.Fatalf("Serve stopped before the restore: %v", err) | |||
| 884 | + | default: | |||
| 885 | + | } | |||
| 886 | + | ||||
| 887 | + | conn, p := attach(t, dir, protocol.IntentAttach, name, sz) | |||
| 888 | + | require.NoError(t, conn.SendClient(protocol.Input{Data: []byte("revived\n")})) | |||
| 889 | + | awaitText(t, p, sz, "revived") | |||
| 890 | + | require.Eventually(t, func() bool { | |||
| 891 | + | sessions := listRemoteSessions(t, dir).Sessions | |||
| 892 | + | return len(sessions) == 1 && sessions[0].State == protocol.SessionUp | |||
| 893 | + | }, 5*time.Second, 10*time.Millisecond, "named session was not restored") | |||
| 894 | + | ||||
| 895 | + | restored, ok, err := opened.Catalogue.Record(name) | |||
| 896 | + | require.NoError(t, err) | |||
| 897 | + | require.True(t, ok) | |||
| 898 | + | require.Equal(t, stopped.IncarnationID, restored.IncarnationID, | |||
| 899 | + | "restore must retain the persisted incarnation") | |||
| 900 | + | require.Equal(t, stopped.CreatedAt, restored.CreatedAt, | |||
| 901 | + | "restore must retain the persisted lifecycle timestamp") | |||
| 902 | + | ||||
| 903 | + | require.NoError(t, conn.Close()) | |||
| 904 | + | select { | |||
| 905 | + | case err := <-served: | |||
| 906 | + | t.Fatalf("Serve stopped while the restored session was live: %v", err) | |||
| 907 | + | default: | |||
| 908 | + | } | |||
| 909 | + | require.NoError(t, killDaemon(dir)) | |||
| 910 | + | select { | |||
| 911 | + | case err := <-served: | |||
| 912 | + | require.NoError(t, err) | |||
| 913 | + | case <-time.After(5 * time.Second): | |||
| 914 | + | t.Fatal("daemon did not stop after kill --daemon") | |||
| 915 | + | } | |||
| 916 | + | } | |||
| 917 | + | ||||
| 724 | func TestMultipleClientsOneLifecycleOwner(t *testing.T) { | 918 | func TestMultipleClientsOneLifecycleOwner(t *testing.T) { | ||
| 725 | dir := filepath.Join(t.TempDir(), "runtime") | 919 | dir := filepath.Join(t.TempDir(), "runtime") | ||
| 726 | owner, err := lifecycle.TryAcquire(dir) | 920 | owner, err := lifecycle.TryAcquire(dir) | ||
| @@ -1495,31 +1689,67 @@ func (*integrationTransport) Close() error { return nil } | |||||
| 1495 | func (*integrationTransport) LocalAddr() net.Addr { return nil } | 1689 | func (*integrationTransport) LocalAddr() net.Addr { return nil } | ||
| 1496 | func (*integrationTransport) RemoteAddr() net.Addr { return nil } | 1690 | func (*integrationTransport) RemoteAddr() net.Addr { return nil } | ||
| 1497 | 1691 | ||||
| 1498 | - | func TestIntegration_KillAllShutsDownDaemon(t *testing.T) { | 1692 | + | // TestIntegration_KillAllPurgesSessionsAndKeepsDaemon is the P4.2 app-level |
| 1693 | + | // contract: `kill --all` removes every live and stopped session but leaves the | |||
| 1694 | + | // daemon serving, and the same process accepts fresh named and ephemeral | |||
| 1695 | + | // sessions that start PTYs, forward input, and open a tab through the ordinary | |||
| 1696 | + | // command palette. Only the distinct explicit daemon stop ends it. | |||
| 1697 | + | func TestIntegration_KillAllPurgesSessionsAndKeepsDaemon(t *testing.T) { | |||
| 1499 | sz := domain.Size{Cols: 80, Rows: 24} | 1698 | sz := domain.Size{Cols: 80, Rows: 24} | ||
| 1500 | - | dir, served := startDaemon(t, daemon.WithShell("/bin/sh", []string{"-c", "sleep 30"})) | 1699 | + | dir, served := startDaemon(t, daemon.WithShell("/bin/cat", nil)) |
| 1501 | - | 1700 | + | ||
| 1502 | - | tr1, _ := attach(t, dir, protocol.IntentNew, "one", sz) | 1701 | + | first, firstPump := attach(t, dir, protocol.IntentNew, "one", sz) |
| 1503 | - | require.NoError(t, tr1.Close()) | 1702 | + | require.NoError(t, first.SendClient(protocol.Input{Data: []byte("alpha\n")})) |
| 1504 | - | tr2, _ := attach(t, dir, protocol.IntentNew, "two", sz) | 1703 | + | awaitText(t, firstPump, sz, "alpha") |
| 1505 | - | require.NoError(t, tr2.Close()) | 1704 | + | // A detached named session becomes stopped authority, so kill-all must purge |
| 1506 | - | 1705 | + | // both the live and the stopped record in one operation. | |
| 1507 | - | killRaw, err := ipc.DialContext(context.Background(), dir) | 1706 | + | second, _ := attach(t, dir, protocol.IntentNew, "two", sz) |
| 1508 | - | require.NoError(t, err) | 1707 | + | require.NoError(t, second.Close()) |
| 1509 | - | killTr := sessionwire.NewClientConnection(killRaw) | 1708 | + | require.Len(t, listRemoteSessions(t, dir).Sessions, 2) |
| 1510 | - | defer func() { _ = killTr.Close() }() | |||
| 1511 | - | require.NoError(t, killTr.SendClient(protocol.Kill{Scope: protocol.KillAll})) | |||
| 1512 | 1709 | ||||
| 1513 | - | _, err = killTr.ReceiveServer() | 1710 | + | require.NoError(t, killAll(dir)) |
| 1514 | - | require.ErrorIs(t, err, io.EOF) | 1711 | + | // The daemon survives the purge: Serve has not returned and the registry is |
| 1515 | - | var serveErr error | 1712 | + | // empty. |
| 1713 | + | select { | |||
| 1714 | + | case err := <-served: | |||
| 1715 | + | t.Fatalf("KillAll stopped Serve: %v", err) | |||
| 1716 | + | default: | |||
| 1717 | + | } | |||
| 1718 | + | require.Eventually(t, func() bool { return len(listRemoteSessions(t, dir).Sessions) == 0 }, | |||
| 1719 | + | 5*time.Second, 10*time.Millisecond, "kill-all did not purge every session") | |||
| 1720 | + | ||||
| 1721 | + | // Reuse: a fresh named session starts its PTY and forwards input. | |||
| 1722 | + | third, thirdPump := attach(t, dir, protocol.IntentNew, "again", sz) | |||
| 1723 | + | require.NoError(t, third.SendClient(protocol.Input{Data: []byte("beta\n")})) | |||
| 1724 | + | awaitText(t, thirdPump, sz, "beta") | |||
| 1725 | + | // A fresh ephemeral session is admitted too. | |||
| 1726 | + | fourth, fourthPump := attach(t, dir, protocol.IntentEphemeral, "", sz) | |||
| 1727 | + | require.NoError(t, fourth.SendClient(protocol.Input{Data: []byte("gamma\n")})) | |||
| 1728 | + | awaitText(t, fourthPump, sz, "gamma") | |||
| 1729 | + | ||||
| 1730 | + | // A tab opened after the purge is real, live topology. | |||
| 1731 | + | require.NoError(t, third.SendClient(protocol.Input{Data: []byte("\x1b ")})) | |||
| 1732 | + | awaitText(t, thirdPump, sz, "Commands") | |||
| 1733 | + | require.NoError(t, third.SendClient(protocol.Input{Data: []byte("CNT\r")})) | |||
| 1516 | require.Eventually(t, func() bool { | 1734 | require.Eventually(t, func() bool { | ||
| 1517 | - | select { | 1735 | + | for _, info := range listRemoteSessions(t, dir).Sessions { |
| 1518 | - | case serveErr = <-served: | 1736 | + | if info.Name == "again" { |
| 1519 | - | return true | 1737 | + | return info.Tabs == 2 |
| 1520 | - | default: | 1738 | + | } |
| 1521 | - | return false | |||
| 1522 | } | 1739 | } | ||
| 1523 | - | }, 15*time.Second, 10*time.Millisecond, "daemon did not shut down after kill all") | 1740 | + | return false |
| 1524 | - | require.NoError(t, serveErr) | 1741 | + | }, 5*time.Second, 10*time.Millisecond, "a tab created after the purge was not published") |
| 1742 | + | ||||
| 1743 | + | _ = first.Close() | |||
| 1744 | + | require.NoError(t, third.Close()) | |||
| 1745 | + | require.NoError(t, fourth.Close()) | |||
| 1746 | + | ||||
| 1747 | + | // Only the distinct explicit daemon stop ends the daemon. | |||
| 1748 | + | require.NoError(t, killDaemon(dir)) | |||
| 1749 | + | select { | |||
| 1750 | + | case err := <-served: | |||
| 1751 | + | require.NoError(t, err) | |||
| 1752 | + | case <-time.After(5 * time.Second): | |||
| 1753 | + | t.Fatal("daemon did not stop after kill --daemon") | |||
| 1754 | + | } | |||
| 1525 | } | 1755 | } | ||
M internal/app/run.go +6 -8
| @@ -135,7 +135,7 @@ usage: | |||
| 135 | 135 | vev host rm <host> remove a pinned remote host | |
| 136 | 136 | vev host list list known remote hosts | |
| 137 | 137 | vev kill <name> kill a session | |
| 138 | - | vev kill --all kill all sessions and stop the daemon | |
| 138 | + | vev kill --all kill all sessions (the daemon keeps running) | |
| 139 | 139 | vev kill --daemon stop the active vev daemon | |
| 140 | 140 | vev cmd <command> run a control command (vev cmd --help) | |
| 141 | 141 | vev --ui-observe expose passive observation for this interactive client | |
| @@ -1771,16 +1771,14 @@ func runKill(ctx context.Context, name string, all, daemon bool) (retErr error) | |||
| 1771 | 1771 | return cause | |
| 1772 | 1772 | } | |
| 1773 | 1773 | } | |
| 1774 | - | if all || daemon { | |
| 1774 | + | // Only an explicit daemon stop ends the daemon; kill-all purges sessions and | |
| 1775 | + | // leaves it running, so it must not wait for an ownership transfer. | |
| 1776 | + | if daemon { | |
| 1775 | 1777 | waitCtx, cancel := context.WithTimeout(ctx, daemonStopTimeout) | |
| 1776 | 1778 | defer cancel() | |
| 1777 | 1779 | owner, waitErr := waitForLifecycleAvailability(waitCtx, ipc.SocketDir(), defaultBackoff) | |
| 1778 | 1780 | if waitErr != nil { | |
| 1779 | - | cause := fmt.Errorf("vev: waiting for daemon ownership transfer: %w", waitErr) | |
| 1780 | - | if daemon { | |
| 1781 | - | return forceStopDaemonFallback(ctx, cause) | |
| 1782 | - | } | |
| 1783 | - | return cause | |
| 1781 | + | return forceStopDaemonFallback(ctx, fmt.Errorf("vev: waiting for daemon ownership transfer: %w", waitErr)) | |
| 1784 | 1782 | } | |
| 1785 | 1783 | if releaseErr := owner.Release(); releaseErr != nil { | |
| 1786 | 1784 | return fmt.Errorf("vev: releasing daemon ownership probe: %w", releaseErr) | |
| @@ -1796,7 +1794,7 @@ func printKillSuccess(name string, all, daemon bool) { | |||
| 1796 | 1794 | return | |
| 1797 | 1795 | } | |
| 1798 | 1796 | if all { | |
| 1799 | - | fmt.Println("killed all sessions and stopped daemon") | |
| 1797 | + | fmt.Println("killed all sessions") | |
| 1800 | 1798 | return | |
| 1801 | 1799 | } | |
| 1802 | 1800 | fmt.Printf("killed %s\n", name) | |
| @@ -135,7 +135,7 @@ usage: | |||||
| 135 | vev host rm <host> remove a pinned remote host | 135 | vev host rm <host> remove a pinned remote host | ||
| 136 | vev host list list known remote hosts | 136 | vev host list list known remote hosts | ||
| 137 | vev kill <name> kill a session | 137 | vev kill <name> kill a session | ||
| 138 | - | vev kill --all kill all sessions and stop the daemon | 138 | + | vev kill --all kill all sessions (the daemon keeps running) |
| 139 | vev kill --daemon stop the active vev daemon | 139 | vev kill --daemon stop the active vev daemon | ||
| 140 | vev cmd <command> run a control command (vev cmd --help) | 140 | vev cmd <command> run a control command (vev cmd --help) | ||
| 141 | vev --ui-observe expose passive observation for this interactive client | 141 | vev --ui-observe expose passive observation for this interactive client | ||
| @@ -1771,16 +1771,14 @@ func runKill(ctx context.Context, name string, all, daemon bool) (retErr error) | |||||
| 1771 | return cause | 1771 | return cause | ||
| 1772 | } | 1772 | } | ||
| 1773 | } | 1773 | } | ||
| 1774 | - | if all || daemon { | 1774 | + | // Only an explicit daemon stop ends the daemon; kill-all purges sessions and |
| 1775 | + | // leaves it running, so it must not wait for an ownership transfer. | |||
| 1776 | + | if daemon { | |||
| 1775 | waitCtx, cancel := context.WithTimeout(ctx, daemonStopTimeout) | 1777 | waitCtx, cancel := context.WithTimeout(ctx, daemonStopTimeout) | ||
| 1776 | defer cancel() | 1778 | defer cancel() | ||
| 1777 | owner, waitErr := waitForLifecycleAvailability(waitCtx, ipc.SocketDir(), defaultBackoff) | 1779 | owner, waitErr := waitForLifecycleAvailability(waitCtx, ipc.SocketDir(), defaultBackoff) | ||
| 1778 | if waitErr != nil { | 1780 | if waitErr != nil { | ||
| 1779 | - | cause := fmt.Errorf("vev: waiting for daemon ownership transfer: %w", waitErr) | 1781 | + | return forceStopDaemonFallback(ctx, fmt.Errorf("vev: waiting for daemon ownership transfer: %w", waitErr)) |
| 1780 | - | if daemon { | |||
| 1781 | - | return forceStopDaemonFallback(ctx, cause) | |||
| 1782 | - | } | |||
| 1783 | - | return cause | |||
| 1784 | } | 1782 | } | ||
| 1785 | if releaseErr := owner.Release(); releaseErr != nil { | 1783 | if releaseErr := owner.Release(); releaseErr != nil { | ||
| 1786 | return fmt.Errorf("vev: releasing daemon ownership probe: %w", releaseErr) | 1784 | return fmt.Errorf("vev: releasing daemon ownership probe: %w", releaseErr) | ||
| @@ -1796,7 +1794,7 @@ func printKillSuccess(name string, all, daemon bool) { | |||||
| 1796 | return | 1794 | return | ||
| 1797 | } | 1795 | } | ||
| 1798 | if all { | 1796 | if all { | ||
| 1799 | - | fmt.Println("killed all sessions and stopped daemon") | 1797 | + | fmt.Println("killed all sessions") |
| 1800 | return | 1798 | return | ||
| 1801 | } | 1799 | } | ||
| 1802 | fmt.Printf("killed %s\n", name) | 1800 | fmt.Printf("killed %s\n", name) | ||
M internal/usecase/daemon/attachment_effect_gate_test.go +4 -1
| @@ -381,7 +381,10 @@ func TestKillSessionInterruptsCapturedSendWithoutClosingNewerIncarnation(t *test | |||
| 381 | 381 | ac.replaceTransport(fresh) | |
| 382 | 382 | } | |
| 383 | 383 | ||
| 384 | - | require.NoError(t, d.killSession(sess, protocol.ReasonSessionKilled, false)) | |
| 384 | + | // The transport incarnation changed after the snapshot, so the kill aborts | |
| 385 | + | // before publishing terminal ownership. That abort must be observable: a | |
| 386 | + | // silent nil would report a teardown that never happened as success. | |
| 387 | + | require.ErrorIs(t, d.killSession(sess, protocol.ReasonSessionKilled, false), errSessionKillParticipantsChanged) | |
| 385 | 388 | awaitTestCompletion(t, sendDone, "captured incarnation attachment effect ticket did not retire") | |
| 386 | 389 | require.True(t, stale.Closed(), "exact captured in-flight transport was not interrupted") | |
| 387 | 390 | require.False(t, fresh.Closed(), "newer transport incarnation was interrupted") | |
| @@ -381,7 +381,10 @@ func TestKillSessionInterruptsCapturedSendWithoutClosingNewerIncarnation(t *test | |||||
| 381 | ac.replaceTransport(fresh) | 381 | ac.replaceTransport(fresh) | ||
| 382 | } | 382 | } | ||
| 383 | 383 | ||||
| 384 | - | require.NoError(t, d.killSession(sess, protocol.ReasonSessionKilled, false)) | 384 | + | // The transport incarnation changed after the snapshot, so the kill aborts |
| 385 | + | // before publishing terminal ownership. That abort must be observable: a | |||
| 386 | + | // silent nil would report a teardown that never happened as success. | |||
| 387 | + | require.ErrorIs(t, d.killSession(sess, protocol.ReasonSessionKilled, false), errSessionKillParticipantsChanged) | |||
| 385 | awaitTestCompletion(t, sendDone, "captured incarnation attachment effect ticket did not retire") | 388 | awaitTestCompletion(t, sendDone, "captured incarnation attachment effect ticket did not retire") | ||
| 386 | require.True(t, stale.Closed(), "exact captured in-flight transport was not interrupted") | 389 | require.True(t, stale.Closed(), "exact captured in-flight transport was not interrupted") | ||
| 387 | require.False(t, fresh.Closed(), "newer transport incarnation was interrupted") | 390 | require.False(t, fresh.Closed(), "newer transport incarnation was interrupted") | ||
M internal/usecase/daemon/client.go +9 -8
| @@ -9,6 +9,15 @@ | |||
| 9 | 9 | // sessionInteractivelyAttachedLocked), but must never freeze, drain, or wait on | |
| 10 | 10 | // a gate while holding any architecture lock. Freeze/drain always runs with | |
| 11 | 11 | // every architecture lock released. | |
| 12 | + | // | |
| 13 | + | // Daemon-level gate order is moveLifecycleMu before d.mu: reserveMoveLifecycles | |
| 14 | + | // holds moveLifecycleMu while taking d.mu to reserve the KillAll purge-gate | |
| 15 | + | // counter, and releaseMoveAdmissionLocked does the same while releasing it. | |
| 16 | + | // closeMoveLifecycles deliberately drops moveLifecycleMu before taking d.mu, so | |
| 17 | + | // no path holds d.mu and then waits on moveLifecycleMu. Consequently | |
| 18 | + | // moveLifecycleReservation.Release must never be called with d.mu held: it takes | |
| 19 | + | // moveLifecycleMu first, releases both the move-gate and purge-gate reservations | |
| 20 | + | // in that order, and only then drops moveLifecycleMu. | |
| 12 | 21 | package daemon | |
| 13 | 22 | ||
| 14 | 23 | import ( | |
| @@ -1120,11 +1129,3 @@ func (d *Daemon) recordClientNotice(sess *session, ac *attachedClient, sev domai | |||
| 1120 | 1129 | func (d *Daemon) resizeForFirstPaint(sess *session, ac *attachedClient, sz domain.Size) bool { | |
| 1121 | 1130 | return sess.geometry.requestResize(d, sess, ac, sz, true) | |
| 1122 | 1131 | } | |
| 1123 | - | ||
| 1124 | - | // killSession removes a session and tears down its resources. It is | |
| 1125 | - | // idempotent: only the caller that wins the registry delete acts. When the | |
| 1126 | - | // registry empties it marks the daemon closing (atomically with the | |
| 1127 | - | // empty-check, under d.mu) and signals shutdown. | |
| 1128 | - | // | |
| 1129 | - | // Teardown ordering matters: context cancel, pty.Close, and the done signal | |
| 1130 | - | // run first and unconditionally — never gated behind a client send. The | |
| @@ -9,6 +9,15 @@ | |||||
| 9 | // sessionInteractivelyAttachedLocked), but must never freeze, drain, or wait on | 9 | // sessionInteractivelyAttachedLocked), but must never freeze, drain, or wait on | ||
| 10 | // a gate while holding any architecture lock. Freeze/drain always runs with | 10 | // a gate while holding any architecture lock. Freeze/drain always runs with | ||
| 11 | // every architecture lock released. | 11 | // every architecture lock released. | ||
| 12 | + | // | |||
| 13 | + | // Daemon-level gate order is moveLifecycleMu before d.mu: reserveMoveLifecycles | |||
| 14 | + | // holds moveLifecycleMu while taking d.mu to reserve the KillAll purge-gate | |||
| 15 | + | // counter, and releaseMoveAdmissionLocked does the same while releasing it. | |||
| 16 | + | // closeMoveLifecycles deliberately drops moveLifecycleMu before taking d.mu, so | |||
| 17 | + | // no path holds d.mu and then waits on moveLifecycleMu. Consequently | |||
| 18 | + | // moveLifecycleReservation.Release must never be called with d.mu held: it takes | |||
| 19 | + | // moveLifecycleMu first, releases both the move-gate and purge-gate reservations | |||
| 20 | + | // in that order, and only then drops moveLifecycleMu. | |||
| 12 | package daemon | 21 | package daemon | ||
| 13 | 22 | ||||
| 14 | import ( | 23 | import ( | ||
| @@ -1120,11 +1129,3 @@ func (d *Daemon) recordClientNotice(sess *session, ac *attachedClient, sev domai | |||||
| 1120 | func (d *Daemon) resizeForFirstPaint(sess *session, ac *attachedClient, sz domain.Size) bool { | 1129 | func (d *Daemon) resizeForFirstPaint(sess *session, ac *attachedClient, sz domain.Size) bool { | ||
| 1121 | return sess.geometry.requestResize(d, sess, ac, sz, true) | 1130 | return sess.geometry.requestResize(d, sess, ac, sz, true) | ||
| 1122 | } | 1131 | } | ||
| 1123 | - | ||||
| 1124 | - | // killSession removes a session and tears down its resources. It is | |||
| 1125 | - | // idempotent: only the caller that wins the registry delete acts. When the | |||
| 1126 | - | // registry empties it marks the daemon closing (atomically with the | |||
| 1127 | - | // empty-check, under d.mu) and signals shutdown. | |||
| 1128 | - | // | |||
| 1129 | - | // Teardown ordering matters: context cancel, pty.Close, and the done signal | |||
| 1130 | - | // run first and unconditionally — never gated behind a client send. The | |||
M internal/usecase/daemon/coderabbit_regression_test.go +3 -3
| @@ -29,7 +29,7 @@ func TestRestoreAmbiguousBadVersionLoadFailurePreservesCheckpoint(t *testing.T) | |||
| 29 | 29 | } | |
| 30 | 30 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 31 | 31 | ||
| 32 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 32 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 33 | 33 | ||
| 34 | 34 | persisted, ok, err := catalogue.Record(record.Name) | |
| 35 | 35 | require.NoError(t, err) | |
| @@ -54,7 +54,7 @@ func TestRestoreTransientLoadFailureBecomesDiscardable(t *testing.T) { | |||
| 54 | 54 | } | |
| 55 | 55 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 56 | 56 | ||
| 57 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 57 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 58 | 58 | ||
| 59 | 59 | persisted, ok, err := catalogue.Record(record.Name) | |
| 60 | 60 | require.NoError(t, err) | |
| @@ -177,7 +177,7 @@ func TestStaleIncompatibleRestoreCannotOverwriteNewerAuthority(t *testing.T) { | |||
| 177 | 177 | d.setStoppedRecovery(replacement, tt.state) | |
| 178 | 178 | } | |
| 179 | 179 | ||
| 180 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 180 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 181 | 181 | ||
| 182 | 182 | persisted, ok, err := catalogue.Record(record.Name) | |
| 183 | 183 | require.NoError(t, err) | |
| @@ -29,7 +29,7 @@ func TestRestoreAmbiguousBadVersionLoadFailurePreservesCheckpoint(t *testing.T) | |||||
| 29 | } | 29 | } | ||
| 30 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 30 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | ||
| 31 | 31 | ||||
| 32 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 32 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 33 | 33 | ||||
| 34 | persisted, ok, err := catalogue.Record(record.Name) | 34 | persisted, ok, err := catalogue.Record(record.Name) | ||
| 35 | require.NoError(t, err) | 35 | require.NoError(t, err) | ||
| @@ -54,7 +54,7 @@ func TestRestoreTransientLoadFailureBecomesDiscardable(t *testing.T) { | |||||
| 54 | } | 54 | } | ||
| 55 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 55 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | ||
| 56 | 56 | ||||
| 57 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 57 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 58 | 58 | ||||
| 59 | persisted, ok, err := catalogue.Record(record.Name) | 59 | persisted, ok, err := catalogue.Record(record.Name) | ||
| 60 | require.NoError(t, err) | 60 | require.NoError(t, err) | ||
| @@ -177,7 +177,7 @@ func TestStaleIncompatibleRestoreCannotOverwriteNewerAuthority(t *testing.T) { | |||||
| 177 | d.setStoppedRecovery(replacement, tt.state) | 177 | d.setStoppedRecovery(replacement, tt.state) | ||
| 178 | } | 178 | } | ||
| 179 | 179 | ||||
| 180 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 180 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 181 | 181 | ||||
| 182 | persisted, ok, err := catalogue.Record(record.Name) | 182 | persisted, ok, err := catalogue.Record(record.Name) | ||
| 183 | require.NoError(t, err) | 183 | require.NoError(t, err) | ||
M internal/usecase/daemon/control.go +22 -0
| @@ -87,6 +87,15 @@ func (d *Daemon) boundedControlSend(tr ports.ServerConnection, message protocol. | |||
| 87 | 87 | return err | |
| 88 | 88 | } | |
| 89 | 89 | ||
| 90 | + | // logControlSendFailure records that an error response could not be delivered. | |
| 91 | + | // A one-shot control client treats EOF as success, so a failed error send would | |
| 92 | + | // otherwise let it infer success from a silent close. | |
| 93 | + | func (d *Daemon) logControlSendFailure(operation string, err error) { | |
| 94 | + | if err != nil { | |
| 95 | + | d.log.Warn("control error response send failed", "operation", operation, "err", err) | |
| 96 | + | } | |
| 97 | + | } | |
| 98 | + | ||
| 90 | 99 | func serverCommandResult(result protocol.CommandResult) protocol.CommandResult { return result } | |
| 91 | 100 | ||
| 92 | 101 | func (d *Daemon) dispatchCommand(ctx context.Context, request protocol.CommandRequest) protocol.CommandResult { | |
| @@ -148,6 +157,16 @@ func (d *Daemon) runControl(cmd command.Command, exec controlExec, request proto | |||
| 148 | 157 | return commandFailure(protocol.ErrInvalidCommandArgs, "usage: "+cmd.Usage) | |
| 149 | 158 | case errors.Is(err, errSessionNameInUse): | |
| 150 | 159 | return commandFailure(protocol.ErrNameTaken, err.Error()) | |
| 160 | + | case errors.Is(err, errPurgeAdmissionClosed): | |
| 161 | + | // A KillAll purge owns the exact lifecycle set. The operation is | |
| 162 | + | // transient (the daemon is not shutting down), so surface a retryable | |
| 163 | + | // internal failure rather than a name or shutdown verdict. | |
| 164 | + | return commandFailure(protocol.ErrInternal, err.Error()) | |
| 165 | + | case errors.Is(err, errSessionKillParticipantsChanged): | |
| 166 | + | // A concurrent detach or token resume invalidated the teardown snapshot | |
| 167 | + | // before it removed anything. The operation is safe to retry, so report | |
| 168 | + | // that clear outcome instead of the internal sentinel text. | |
| 169 | + | return commandFailure(protocol.ErrInternal, sessionKillRetryMessage) | |
| 151 | 170 | case isMoveCommandError(err): | |
| 152 | 171 | return moveCommandFailure(err) | |
| 153 | 172 | case errors.Is(err, layout.ErrNotInSplit): | |
| @@ -490,6 +509,9 @@ func (e controlExec) CreateSessionNamed(name string) error { | |||
| 490 | 509 | if e.d.closing { | |
| 491 | 510 | return errors.New("daemon is shutting down") | |
| 492 | 511 | } | |
| 512 | + | if e.d.purgeAdmissionClosedLocked() { | |
| 513 | + | return errPurgeAdmissionClosed | |
| 514 | + | } | |
| 493 | 515 | if e.d.nameLiveOrStoppedLocked(name) { | |
| 494 | 516 | return errSessionNameInUse | |
| 495 | 517 | } | |
| @@ -87,6 +87,15 @@ func (d *Daemon) boundedControlSend(tr ports.ServerConnection, message protocol. | |||||
| 87 | return err | 87 | return err | ||
| 88 | } | 88 | } | ||
| 89 | 89 | ||||
| 90 | + | // logControlSendFailure records that an error response could not be delivered. | |||
| 91 | + | // A one-shot control client treats EOF as success, so a failed error send would | |||
| 92 | + | // otherwise let it infer success from a silent close. | |||
| 93 | + | func (d *Daemon) logControlSendFailure(operation string, err error) { | |||
| 94 | + | if err != nil { | |||
| 95 | + | d.log.Warn("control error response send failed", "operation", operation, "err", err) | |||
| 96 | + | } | |||
| 97 | + | } | |||
| 98 | + | ||||
| 90 | func serverCommandResult(result protocol.CommandResult) protocol.CommandResult { return result } | 99 | func serverCommandResult(result protocol.CommandResult) protocol.CommandResult { return result } | ||
| 91 | 100 | ||||
| 92 | func (d *Daemon) dispatchCommand(ctx context.Context, request protocol.CommandRequest) protocol.CommandResult { | 101 | func (d *Daemon) dispatchCommand(ctx context.Context, request protocol.CommandRequest) protocol.CommandResult { | ||
| @@ -148,6 +157,16 @@ func (d *Daemon) runControl(cmd command.Command, exec controlExec, request proto | |||||
| 148 | return commandFailure(protocol.ErrInvalidCommandArgs, "usage: "+cmd.Usage) | 157 | return commandFailure(protocol.ErrInvalidCommandArgs, "usage: "+cmd.Usage) | ||
| 149 | case errors.Is(err, errSessionNameInUse): | 158 | case errors.Is(err, errSessionNameInUse): | ||
| 150 | return commandFailure(protocol.ErrNameTaken, err.Error()) | 159 | return commandFailure(protocol.ErrNameTaken, err.Error()) | ||
| 160 | + | case errors.Is(err, errPurgeAdmissionClosed): | |||
| 161 | + | // A KillAll purge owns the exact lifecycle set. The operation is | |||
| 162 | + | // transient (the daemon is not shutting down), so surface a retryable | |||
| 163 | + | // internal failure rather than a name or shutdown verdict. | |||
| 164 | + | return commandFailure(protocol.ErrInternal, err.Error()) | |||
| 165 | + | case errors.Is(err, errSessionKillParticipantsChanged): | |||
| 166 | + | // A concurrent detach or token resume invalidated the teardown snapshot | |||
| 167 | + | // before it removed anything. The operation is safe to retry, so report | |||
| 168 | + | // that clear outcome instead of the internal sentinel text. | |||
| 169 | + | return commandFailure(protocol.ErrInternal, sessionKillRetryMessage) | |||
| 151 | case isMoveCommandError(err): | 170 | case isMoveCommandError(err): | ||
| 152 | return moveCommandFailure(err) | 171 | return moveCommandFailure(err) | ||
| 153 | case errors.Is(err, layout.ErrNotInSplit): | 172 | case errors.Is(err, layout.ErrNotInSplit): | ||
| @@ -490,6 +509,9 @@ func (e controlExec) CreateSessionNamed(name string) error { | |||||
| 490 | if e.d.closing { | 509 | if e.d.closing { | ||
| 491 | return errors.New("daemon is shutting down") | 510 | return errors.New("daemon is shutting down") | ||
| 492 | } | 511 | } | ||
| 512 | + | if e.d.purgeAdmissionClosedLocked() { | |||
| 513 | + | return errPurgeAdmissionClosed | |||
| 514 | + | } | |||
| 493 | if e.d.nameLiveOrStoppedLocked(name) { | 515 | if e.d.nameLiveOrStoppedLocked(name) { | ||
| 494 | return errSessionNameInUse | 516 | return errSessionNameInUse | ||
| 495 | } | 517 | } | ||
M internal/usecase/daemon/control_test.go +85 -0
| @@ -995,6 +995,25 @@ func TestHandleCommandMovePaneUsesActiveFocusedSourceWithSessionFlag(t *testing. | |||
| 995 | 995 | destTab.mu.Unlock() | |
| 996 | 996 | } | |
| 997 | 997 | ||
| 998 | + | // TestHandleCommandMovePanePurgeAdmissionMapsToRetryableError proves a move | |
| 999 | + | // command rejected because a transient KillAll purge owns the exact lifecycle | |
| 1000 | + | // set surfaces the same retryable internal outcome as the create path, instead | |
| 1001 | + | // of misreporting the destination as gone. | |
| 1002 | + | func TestHandleCommandMovePanePurgeAdmissionMapsToRetryableError(t *testing.T) { | |
| 1003 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 1004 | + | addControlSession(d, "work", "t_work", "p_work") | |
| 1005 | + | addNamedMoveDestination(d, "dest", "t_dest", "p_dest") | |
| 1006 | + | ||
| 1007 | + | beginTestPurgeAdmission(t, d) | |
| 1008 | + | result := sendCommand(t, d, protocol.CommandRequest{ | |
| 1009 | + | Slug: "move-pane", Args: []string{"dest", "t_dest"}, TargetSession: "work", | |
| 1010 | + | }) | |
| 1011 | + | require.False(t, result.OK) | |
| 1012 | + | require.Equal(t, protocol.ErrInternal, result.Code) | |
| 1013 | + | require.Equal(t, errPurgeAdmissionClosed.Error(), result.Text) | |
| 1014 | + | d.endPurgeAdmission() | |
| 1015 | + | } | |
| 1016 | + | ||
| 998 | 1017 | func TestHandleCommandMovePaneSelfUsesStableSourceIDs(t *testing.T) { | |
| 999 | 1018 | d := newTestDaemon(t, nil, stubClock{}) | |
| 1000 | 1019 | source := addControlSession(d, "work", "t_active", "p_active") | |
| @@ -1441,3 +1460,69 @@ func addControlSession(d *Daemon, name, tabID, paneID string) *session { | |||
| 1441 | 1460 | d.mu.Unlock() | |
| 1442 | 1461 | return sess | |
| 1443 | 1462 | } | |
| 1463 | + | ||
| 1464 | + | // TestHandleKillSessionParticipantsChangedReportsRetryable proves a direct | |
| 1465 | + | // session kill whose teardown snapshot a concurrent token resume invalidated | |
| 1466 | + | // reports a clear retryable outcome instead of a permanent delete failure, and | |
| 1467 | + | // leaves the session live. | |
| 1468 | + | func TestHandleKillSessionParticipantsChangedReportsRetryable(t *testing.T) { | |
| 1469 | + | d, sess, ac, _ := newManualSessionWithPTYs(t, newQuietPTY()) | |
| 1470 | + | ||
| 1471 | + | d.afterAttachmentEffectParticipantsSnapshotted = func(action string, _ []*attachedClient) { | |
| 1472 | + | if action != "" { | |
| 1473 | + | return | |
| 1474 | + | } | |
| 1475 | + | d.afterAttachmentEffectParticipantsSnapshotted = nil | |
| 1476 | + | // A token resume replaces the exact transport after the participant | |
| 1477 | + | // snapshot was taken, so the kill aborts before removing anything. | |
| 1478 | + | fresh := &closeTrackingTransport{} | |
| 1479 | + | ac.replaceTransport(fresh) | |
| 1480 | + | ac.installTestAttachmentCapability(sess.captureAttachmentCapability(ac, fresh)) | |
| 1481 | + | } | |
| 1482 | + | t.Cleanup(func() { d.afterAttachmentEffectParticipantsSnapshotted = nil }) | |
| 1483 | + | ||
| 1484 | + | tr, sends := newCapturingTransport(t) | |
| 1485 | + | d.handleKill(tr, protocol.Kill{Scope: protocol.KillSession, Name: "work"}) | |
| 1486 | + | ||
| 1487 | + | frame := awaitFrame(t, sends, "Error") | |
| 1488 | + | msg, ok := decodeServerMessage(t, frame).(protocol.ErrorMsg) | |
| 1489 | + | require.True(t, ok) | |
| 1490 | + | require.Equal(t, protocol.ErrInternal, msg.Code) | |
| 1491 | + | require.Equal(t, sessionKillRetryMessage, msg.Text) | |
| 1492 | + | require.Equal(t, 1, sessionCount(d), "an aborted direct kill must leave the session live") | |
| 1493 | + | require.Same(t, sess, firstSession(d)) | |
| 1494 | + | } | |
| 1495 | + | ||
| 1496 | + | // TestCloseTabParticipantsChangedReportsRetryable proves a last-tab close whose | |
| 1497 | + | // teardown snapshot a concurrent token resume invalidated reports a clear | |
| 1498 | + | // retryable outcome to both the command result and the attached client, and | |
| 1499 | + | // leaves the session and its tab live. | |
| 1500 | + | func TestCloseTabParticipantsChangedReportsRetryable(t *testing.T) { | |
| 1501 | + | d, sess, ac, _ := newManualSessionWithPTYs(t, newQuietPTY()) | |
| 1502 | + | ||
| 1503 | + | d.afterAttachmentEffectParticipantsSnapshotted = func(action string, _ []*attachedClient) { | |
| 1504 | + | if action != "" { | |
| 1505 | + | return | |
| 1506 | + | } | |
| 1507 | + | d.afterAttachmentEffectParticipantsSnapshotted = nil | |
| 1508 | + | fresh := &closeTrackingTransport{} | |
| 1509 | + | ac.replaceTransport(fresh) | |
| 1510 | + | ac.installTestAttachmentCapability(sess.captureAttachmentCapability(ac, fresh)) | |
| 1511 | + | } | |
| 1512 | + | t.Cleanup(func() { d.afterAttachmentEffectParticipantsSnapshotted = nil }) | |
| 1513 | + | ||
| 1514 | + | result := sendCommand(t, d, protocol.CommandRequest{Slug: "close-tab", TargetSession: "work"}) | |
| 1515 | + | require.False(t, result.OK) | |
| 1516 | + | require.Equal(t, protocol.ErrInternal, result.Code) | |
| 1517 | + | require.Equal(t, sessionKillRetryMessage, result.Text) | |
| 1518 | + | ||
| 1519 | + | toasts := awaitToastCount(t, ac, 1) | |
| 1520 | + | require.Contains(t, toasts[0].Message, "couldn't close tab") | |
| 1521 | + | require.Contains(t, toasts[0].Message, "try again") | |
| 1522 | + | ||
| 1523 | + | sess.mu.Lock() | |
| 1524 | + | tabs := len(sess.tabs) | |
| 1525 | + | sess.mu.Unlock() | |
| 1526 | + | require.Equal(t, 1, tabs, "an aborted last-tab close must retain the tab") | |
| 1527 | + | require.Equal(t, 1, sessionCount(d), "an aborted last-tab close must retain the session") | |
| 1528 | + | } | |
| @@ -995,6 +995,25 @@ func TestHandleCommandMovePaneUsesActiveFocusedSourceWithSessionFlag(t *testing. | |||||
| 995 | destTab.mu.Unlock() | 995 | destTab.mu.Unlock() | ||
| 996 | } | 996 | } | ||
| 997 | 997 | ||||
| 998 | + | // TestHandleCommandMovePanePurgeAdmissionMapsToRetryableError proves a move | |||
| 999 | + | // command rejected because a transient KillAll purge owns the exact lifecycle | |||
| 1000 | + | // set surfaces the same retryable internal outcome as the create path, instead | |||
| 1001 | + | // of misreporting the destination as gone. | |||
| 1002 | + | func TestHandleCommandMovePanePurgeAdmissionMapsToRetryableError(t *testing.T) { | |||
| 1003 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 1004 | + | addControlSession(d, "work", "t_work", "p_work") | |||
| 1005 | + | addNamedMoveDestination(d, "dest", "t_dest", "p_dest") | |||
| 1006 | + | ||||
| 1007 | + | beginTestPurgeAdmission(t, d) | |||
| 1008 | + | result := sendCommand(t, d, protocol.CommandRequest{ | |||
| 1009 | + | Slug: "move-pane", Args: []string{"dest", "t_dest"}, TargetSession: "work", | |||
| 1010 | + | }) | |||
| 1011 | + | require.False(t, result.OK) | |||
| 1012 | + | require.Equal(t, protocol.ErrInternal, result.Code) | |||
| 1013 | + | require.Equal(t, errPurgeAdmissionClosed.Error(), result.Text) | |||
| 1014 | + | d.endPurgeAdmission() | |||
| 1015 | + | } | |||
| 1016 | + | ||||
| 998 | func TestHandleCommandMovePaneSelfUsesStableSourceIDs(t *testing.T) { | 1017 | func TestHandleCommandMovePaneSelfUsesStableSourceIDs(t *testing.T) { | ||
| 999 | d := newTestDaemon(t, nil, stubClock{}) | 1018 | d := newTestDaemon(t, nil, stubClock{}) | ||
| 1000 | source := addControlSession(d, "work", "t_active", "p_active") | 1019 | source := addControlSession(d, "work", "t_active", "p_active") | ||
| @@ -1441,3 +1460,69 @@ func addControlSession(d *Daemon, name, tabID, paneID string) *session { | |||||
| 1441 | d.mu.Unlock() | 1460 | d.mu.Unlock() | ||
| 1442 | return sess | 1461 | return sess | ||
| 1443 | } | 1462 | } | ||
| 1463 | + | ||||
| 1464 | + | // TestHandleKillSessionParticipantsChangedReportsRetryable proves a direct | |||
| 1465 | + | // session kill whose teardown snapshot a concurrent token resume invalidated | |||
| 1466 | + | // reports a clear retryable outcome instead of a permanent delete failure, and | |||
| 1467 | + | // leaves the session live. | |||
| 1468 | + | func TestHandleKillSessionParticipantsChangedReportsRetryable(t *testing.T) { | |||
| 1469 | + | d, sess, ac, _ := newManualSessionWithPTYs(t, newQuietPTY()) | |||
| 1470 | + | ||||
| 1471 | + | d.afterAttachmentEffectParticipantsSnapshotted = func(action string, _ []*attachedClient) { | |||
| 1472 | + | if action != "" { | |||
| 1473 | + | return | |||
| 1474 | + | } | |||
| 1475 | + | d.afterAttachmentEffectParticipantsSnapshotted = nil | |||
| 1476 | + | // A token resume replaces the exact transport after the participant | |||
| 1477 | + | // snapshot was taken, so the kill aborts before removing anything. | |||
| 1478 | + | fresh := &closeTrackingTransport{} | |||
| 1479 | + | ac.replaceTransport(fresh) | |||
| 1480 | + | ac.installTestAttachmentCapability(sess.captureAttachmentCapability(ac, fresh)) | |||
| 1481 | + | } | |||
| 1482 | + | t.Cleanup(func() { d.afterAttachmentEffectParticipantsSnapshotted = nil }) | |||
| 1483 | + | ||||
| 1484 | + | tr, sends := newCapturingTransport(t) | |||
| 1485 | + | d.handleKill(tr, protocol.Kill{Scope: protocol.KillSession, Name: "work"}) | |||
| 1486 | + | ||||
| 1487 | + | frame := awaitFrame(t, sends, "Error") | |||
| 1488 | + | msg, ok := decodeServerMessage(t, frame).(protocol.ErrorMsg) | |||
| 1489 | + | require.True(t, ok) | |||
| 1490 | + | require.Equal(t, protocol.ErrInternal, msg.Code) | |||
| 1491 | + | require.Equal(t, sessionKillRetryMessage, msg.Text) | |||
| 1492 | + | require.Equal(t, 1, sessionCount(d), "an aborted direct kill must leave the session live") | |||
| 1493 | + | require.Same(t, sess, firstSession(d)) | |||
| 1494 | + | } | |||
| 1495 | + | ||||
| 1496 | + | // TestCloseTabParticipantsChangedReportsRetryable proves a last-tab close whose | |||
| 1497 | + | // teardown snapshot a concurrent token resume invalidated reports a clear | |||
| 1498 | + | // retryable outcome to both the command result and the attached client, and | |||
| 1499 | + | // leaves the session and its tab live. | |||
| 1500 | + | func TestCloseTabParticipantsChangedReportsRetryable(t *testing.T) { | |||
| 1501 | + | d, sess, ac, _ := newManualSessionWithPTYs(t, newQuietPTY()) | |||
| 1502 | + | ||||
| 1503 | + | d.afterAttachmentEffectParticipantsSnapshotted = func(action string, _ []*attachedClient) { | |||
| 1504 | + | if action != "" { | |||
| 1505 | + | return | |||
| 1506 | + | } | |||
| 1507 | + | d.afterAttachmentEffectParticipantsSnapshotted = nil | |||
| 1508 | + | fresh := &closeTrackingTransport{} | |||
| 1509 | + | ac.replaceTransport(fresh) | |||
| 1510 | + | ac.installTestAttachmentCapability(sess.captureAttachmentCapability(ac, fresh)) | |||
| 1511 | + | } | |||
| 1512 | + | t.Cleanup(func() { d.afterAttachmentEffectParticipantsSnapshotted = nil }) | |||
| 1513 | + | ||||
| 1514 | + | result := sendCommand(t, d, protocol.CommandRequest{Slug: "close-tab", TargetSession: "work"}) | |||
| 1515 | + | require.False(t, result.OK) | |||
| 1516 | + | require.Equal(t, protocol.ErrInternal, result.Code) | |||
| 1517 | + | require.Equal(t, sessionKillRetryMessage, result.Text) | |||
| 1518 | + | ||||
| 1519 | + | toasts := awaitToastCount(t, ac, 1) | |||
| 1520 | + | require.Contains(t, toasts[0].Message, "couldn't close tab") | |||
| 1521 | + | require.Contains(t, toasts[0].Message, "try again") | |||
| 1522 | + | ||||
| 1523 | + | sess.mu.Lock() | |||
| 1524 | + | tabs := len(sess.tabs) | |||
| 1525 | + | sess.mu.Unlock() | |||
| 1526 | + | require.Equal(t, 1, tabs, "an aborted last-tab close must retain the tab") | |||
| 1527 | + | require.Equal(t, 1, sessionCount(d), "an aborted last-tab close must retain the session") | |||
| 1528 | + | } | |||
M internal/usecase/daemon/daemon.go +128 -63
| @@ -91,11 +91,13 @@ type Daemon struct { | |||
| 91 | 91 | lastAllocatedCreatedAt int64 | |
| 92 | 92 | mruSeq atomic.Uint64 | |
| 93 | 93 | creationRequestSeq atomic.Uint64 | |
| 94 | - | // closing marks that shutdown has irreversibly begun. It is set under mu, | |
| 95 | - | // atomically with the event that makes shutdown inevitable (the registry | |
| 96 | - | // emptying in killSession, or shutdownAll starting), and checked by route | |
| 97 | - | // under the same mutex — so a Hello racing shutdown can never insert a new | |
| 98 | - | // session that nobody would tear down. | |
| 94 | + | // closing marks that explicit daemon shutdown (KillDaemon/process | |
| 95 | + | // cancellation) has irreversibly begun. It is set under mu when | |
| 96 | + | // shutdownAll starts, and checked by route under the same mutex — so a Hello | |
| 97 | + | // racing shutdown can never insert a new session that nobody would tear | |
| 98 | + | // down. Neither an empty registry (final session removal) nor a KillAll | |
| 99 | + | // purge sets it: the daemon survives empty→occupied→empty and survives a | |
| 100 | + | // purge, and only an explicit stop/process cancellation ends it. | |
| 99 | 101 | closing bool | |
| 100 | 102 | ||
| 101 | 103 | // moveLifecycleMu is the daemon-level admission gate for transferable | |
| @@ -110,6 +112,28 @@ type Daemon struct { | |||
| 110 | 112 | paneProcessCtx context.Context | |
| 111 | 113 | paneProcessCancel context.CancelFunc | |
| 112 | 114 | ||
| 115 | + | // purgeAllMu grants one daemon-wide purge (KillAll) owner at a time so two | |
| 116 | + | // concurrent kill-all requests never interleave their exact lifecycle sets. | |
| 117 | + | // It is not an admission gate; see purgeAdmissionClosing below. | |
| 118 | + | purgeAllMu sync.Mutex | |
| 119 | + | // purgeAdmissionClosing is the transient KillAll admission gate. Unlike | |
| 120 | + | // closing and moveLifecycleClosing it is re-openable: purgeAllSessions sets | |
| 121 | + | // it under mu, removes exactly the live and stopped/broken/durable records | |
| 122 | + | // captured at that instant, then clears it so the daemon keeps serving. | |
| 123 | + | // Creation and restoration wait on the changed channel (a blocked writer | |
| 124 | + | // keeps new admissions out), while moves and internal creations reject, so | |
| 125 | + | // no transition can be inserted into the purge's exact lifecycle set. | |
| 126 | + | // purgeAdmissionActive counts admitted transitions the purge must drain. | |
| 127 | + | // purgeEpoch increments once at each purge that acquires the gate (after the | |
| 128 | + | // drain) so a restoration worker blocked on admission can detect a purge | |
| 129 | + | // that completed while it waited and must not resurrect the records captured | |
| 130 | + | // before that purge. | |
| 131 | + | // All three fields are guarded by mu. | |
| 132 | + | purgeAdmissionClosing bool | |
| 133 | + | purgeAdmissionActive uint | |
| 134 | + | purgeAdmissionChanged chan struct{} | |
| 135 | + | purgeEpoch uint64 | |
| 136 | + | ||
| 113 | 137 | // notifies holds one completion channel per in-flight async Detached | |
| 114 | 138 | // notification (guarded by mu, pruned on insert). Channels rather than a | |
| 115 | 139 | // WaitGroup: notifications are spawned from arbitrary goroutines while | |
| @@ -180,6 +204,21 @@ type Daemon struct { | |||
| 180 | 204 | // afterAttachmentEffectsFrozen observes the lock-free boundary after all affected | |
| 181 | 205 | // attachment gates are frozen and drained, before architecture publication. | |
| 182 | 206 | afterAttachmentEffectsFrozen func() | |
| 207 | + | // afterPurgeAdmission is a deterministic test seam that runs after a KillAll | |
| 208 | + | // purge has closed purge admission and taken the closing fence, while it | |
| 209 | + | // still owns purgeAllMu. It lets tests observe purge serialization. | |
| 210 | + | afterPurgeAdmission func() | |
| 211 | + | // beforePurgeLiveSessionKill is a deterministic test seam that runs after a | |
| 212 | + | // KillAll purge's per-unit shutdown precheck and before that live unit | |
| 213 | + | // attempts teardown ownership. Tests use it to let an explicit stop win that | |
| 214 | + | // exact window, proving a stop-owned unit is never reported as a success. | |
| 215 | + | beforePurgeLiveSessionKill func(*session) | |
| 216 | + | // afterSessionKillQuarantine is a deterministic test seam that runs while a | |
| 217 | + | // teardown owns the session and its snapshot coordinator is quarantined, but | |
| 218 | + | // before the pre-publication revalidation. Tests use it to change the | |
| 219 | + | // participant set in that window and prove a post-quarantine abort rolls the | |
| 220 | + | // quarantine back so a surviving session keeps checkpointing. | |
| 221 | + | afterSessionKillQuarantine func(*session) | |
| 183 | 222 | // beforeAttachmentTransitionIdentityAdmission is a deterministic test seam | |
| 184 | 223 | // after a ready transition published its capability and before the committed | |
| 185 | 224 | // route identity is admitted, so tests can supersede that exact capability in | |
| @@ -348,6 +387,10 @@ type Daemon struct { | |||
| 348 | 387 | hardCtx context.Context | |
| 349 | 388 | hardCancel context.CancelFunc | |
| 350 | 389 | ||
| 390 | + | // done is closed exactly once by an explicit daemon stop (shutdownAll) to | |
| 391 | + | // break Serve's accept loop. It is deliberately not closed by final session | |
| 392 | + | // removal or by a KillAll purge: an empty registry and a purged registry | |
| 393 | + | // both keep the daemon serving. | |
| 351 | 394 | done chan struct{} | |
| 352 | 395 | doneOnce sync.Once | |
| 353 | 396 | ||
| @@ -447,6 +490,10 @@ func WithRuntimeObserver(observer ports.SerializedRuntimeObserver) Option { | |||
| 447 | 490 | // WithRemoteMonitor installs the snapshot-only remote directory and the | |
| 448 | 491 | // monitor runner. Serve starts the runner without waiting for registry, | |
| 449 | 492 | // cache or runtime readiness and stops it with its own context. | |
| 493 | + | // | |
| 494 | + | // The daemon-side monitor stays intentionally active in this state: it is part | |
| 495 | + | // of the coordinated P7 removal set, so no polling gate, feature switch, or | |
| 496 | + | // early shutdown path is added here (GO-001, deferred to P7). | |
| 450 | 497 | func WithRemoteMonitor(directory ports.RemoteDirectory, run func(context.Context) error) Option { | |
| 451 | 498 | return func(d *Daemon) { | |
| 452 | 499 | d.remoteDirectory = directory | |
| @@ -792,8 +839,12 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||
| 792 | 839 | const remoteMonitorShutdownJoin = 3 * time.Second | |
| 793 | 840 | ||
| 794 | 841 | // Serve runs the accept loop over l, owning it for the loop's lifetime. It | |
| 795 | - | // returns when the last session is removed or ctx is cancelled; on the latter | |
| 796 | - | // path attached clients are detached with ReasonServerShutdown. | |
| 842 | + | // returns only on ctx cancellation or an explicit daemon shutdown request | |
| 843 | + | // (KillDaemon), never when the session registry drains to empty: final | |
| 844 | + | // session removal is independent of daemon shutdown so the daemon survives | |
| 845 | + | // empty→occupied→empty and can accept fresh sessions again. A KillAll purge | |
| 846 | + | // leaves Serve running too. On the ctx path | |
| 847 | + | // attached clients are detached with ReasonServerShutdown. | |
| 797 | 848 | func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | |
| 798 | 849 | d.serveCtx, d.serveCancel = context.WithCancel(ctx) | |
| 799 | 850 | defer d.serveCancel() | |
| @@ -866,8 +917,9 @@ func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | |||
| 866 | 917 | } else { | |
| 867 | 918 | d.closeRestoreDone() | |
| 868 | 919 | } | |
| 869 | - | // Break the accept loop when either the parent context is cancelled or the | |
| 870 | - | // registry drains to empty: both close the listener, which fails Accept. | |
| 920 | + | // Break the accept loop only on an explicit shutdown: the parent context | |
| 921 | + | // being cancelled or an explicit daemon stop request (d.done). An empty | |
| 922 | + | // registry never closes the listener, so the daemon keeps accepting. | |
| 871 | 923 | go func() { | |
| 872 | 924 | select { | |
| 873 | 925 | case <-d.serveCtx.Done(): | |
| @@ -908,7 +960,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | |||
| 908 | 960 | snapshotDeadline = newSnapshotShutdownDeadline(d.clock) | |
| 909 | 961 | defer snapshotDeadline.stop() | |
| 910 | 962 | } | |
| 911 | - | d.terminateAllWithSnapshotDeadline(protocol.ReasonServerShutdown, false, snapshotDeadline) | |
| 963 | + | d.terminateAllForShutdown(protocol.ReasonServerShutdown, snapshotDeadline) | |
| 912 | 964 | d.waitNotifies() | |
| 913 | 965 | d.hardCancel() | |
| 914 | 966 | d.serveCancel() | |
| @@ -928,7 +980,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | |||
| 928 | 980 | d.persistShutdownSnapshotFailure(name, context.DeadlineExceeded) | |
| 929 | 981 | } | |
| 930 | 982 | d.WaitDurableWriters() | |
| 931 | - | d.terminateAllWithSnapshotDeadline(protocol.ReasonServerShutdown, false, snapshotDeadline) | |
| 983 | + | d.terminateAllForShutdown(protocol.ReasonServerShutdown, snapshotDeadline) | |
| 932 | 984 | d.waitSessionWorkersWithSnapshotDeadline(snapshotDeadline) | |
| 933 | 985 | d.waitNotifies() | |
| 934 | 986 | if err := d.flushCatalogue(); err != nil { | |
| @@ -942,58 +994,41 @@ func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | |||
| 942 | 994 | return nil | |
| 943 | 995 | } | |
| 944 | 996 | ||
| 945 | - | // shutdownAll marks the daemon closing and kills every live session. Setting | |
| 946 | - | // closing under the same lock as the snapshot guarantees no session can be | |
| 947 | - | // inserted after the snapshot: route rejects once closing is set, and both run | |
| 948 | - | // under d.mu. killSession (which relocks) runs after the lock is released. | |
| 997 | + | // shutdownAll is the explicit daemon-stop operation (wire KillDaemon). It is | |
| 998 | + | // the only caller that irreversibly ends the daemon: it closes move admission, | |
| 999 | + | // marks closing, cancels the daemon-wide pane process context, preserves every | |
| 1000 | + | // live session as stopped durable authority, and closes done so Serve returns. | |
| 1001 | + | // KillAll never reaches this path. Setting closing under the same lock as the | |
| 1002 | + | // snapshot guarantees no session can be inserted after the snapshot: route | |
| 1003 | + | // rejects once closing is set, and both run under d.mu. killSession (which | |
| 1004 | + | // relocks) runs after the lock is released. | |
| 949 | 1005 | func (d *Daemon) shutdownAll(reason uint8) (checkpointIncomplete bool) { | |
| 950 | - | return d.terminateAllWithSnapshotDeadline(reason, false, nil) | |
| 951 | - | } | |
| 952 | - | ||
| 953 | - | func (d *Daemon) purgeAll(reason uint8) (checkpointIncomplete bool) { | |
| 954 | - | return d.terminateAllWithSnapshotDeadline(reason, true, nil) | |
| 1006 | + | return d.terminateAllForShutdown(reason, nil) | |
| 955 | 1007 | } | |
| 956 | 1008 | ||
| 957 | - | func (d *Daemon) terminateAllWithSnapshotDeadline(reason uint8, purge bool, deadline *snapshotShutdownDeadline) (checkpointIncomplete bool) { | |
| 1009 | + | // terminateAllForShutdown performs the irreversible daemon teardown shared by | |
| 1010 | + | // shutdownAll and Serve's exit path. It is separate from purgeAllSessions, | |
| 1011 | + | // which leaves every daemon-wide shutdown fact untouched. | |
| 1012 | + | func (d *Daemon) terminateAllForShutdown(reason uint8, deadline *snapshotShutdownDeadline) (checkpointIncomplete bool) { | |
| 958 | 1013 | d.closeMoveLifecycles() | |
| 959 | 1014 | d.mu.Lock() | |
| 960 | 1015 | d.closing = true | |
| 1016 | + | // Wake any KillAll waiting on purge admission so it defers to this stop | |
| 1017 | + | // instead of holding the gate until its bounded drain deadline expires. | |
| 1018 | + | d.signalPurgeAdmissionChangedLocked() | |
| 961 | 1019 | d.purgeAllParkingLocked() | |
| 962 | 1020 | parkedRetirements := d.purgeAllParkedLocked() | |
| 963 | 1021 | d.purgeAllSuspendedLocked() | |
| 964 | 1022 | snapshot := d.sessionsSnapshotLocked() | |
| 965 | - | stoppedNames := make([]string, 0, len(d.inactive)) | |
| 966 | - | if purge { | |
| 967 | - | for name := range d.inactive { | |
| 968 | - | stoppedNames = append(stoppedNames, name) | |
| 969 | - | } | |
| 970 | - | sort.Strings(stoppedNames) | |
| 971 | - | } | |
| 972 | - | empty := len(snapshot) == 0 | |
| 973 | 1023 | d.mu.Unlock() | |
| 974 | 1024 | d.finishParkedAttachmentRetirements(parkedRetirements) | |
| 975 | - | if !purge { | |
| 976 | - | // Publish preservation policy for the complete registry snapshot before | |
| 977 | - | // cancelling any PTY. Cancellation-driven EOF is allowed to own teardown, | |
| 978 | - | // but its disposition must remain daemon shutdown rather than session purge. | |
| 979 | - | for _, s := range snapshot { | |
| 980 | - | s.reserveShutdownTeardown() | |
| 981 | - | } | |
| 982 | - | } | |
| 983 | - | d.log.Info("session termination begin", "reason", reason, "purge", purge, "live_sessions", len(snapshot), "stopped_sessions", len(stoppedNames)) | |
| 984 | - | // Purge stopped authority before signaling Serve to exit. Once the final | |
| 985 | - | // live registry entry closes, Serve cancels its context; deleting stopped | |
| 986 | - | // records afterward would be interrupted and retain broken catalogue entries. | |
| 987 | - | for _, name := range stoppedNames { | |
| 988 | - | if err := d.retryStoppedPurge(name); err != nil { | |
| 989 | - | checkpointIncomplete = true | |
| 990 | - | d.log.Error("deleting stopped session during kill all failed", "err", err, "session", name) | |
| 991 | - | } | |
| 992 | - | } | |
| 993 | - | if empty { | |
| 994 | - | d.doneOnce.Do(func() { close(d.done) }) | |
| 995 | - | return checkpointIncomplete | |
| 1025 | + | // Publish preservation policy for the complete registry snapshot before | |
| 1026 | + | // cancelling any PTY. Cancellation-driven EOF is allowed to own teardown, | |
| 1027 | + | // but its disposition must remain daemon shutdown rather than session purge. | |
| 1028 | + | for _, s := range snapshot { | |
| 1029 | + | s.reserveShutdownTeardown() | |
| 996 | 1030 | } | |
| 1031 | + | d.log.Info("session termination begin", "reason", reason, "live_sessions", len(snapshot)) | |
| 997 | 1032 | for _, s := range snapshot { | |
| 998 | 1033 | // Cancellation and PTY closure must not wait behind a teardown owner that | |
| 999 | 1034 | // is blocked in snapshot publication or purge work. | |
| @@ -1002,11 +1037,11 @@ func (d *Daemon) terminateAllWithSnapshotDeadline(reason uint8, purge bool, dead | |||
| 1002 | 1037 | ephemeral := s.ephemeral | |
| 1003 | 1038 | s.mu.Unlock() | |
| 1004 | 1039 | s.stopInMemoryLifecycle() | |
| 1005 | - | if err := d.killSessionWithSnapshotDeadline(s, reason, purge, deadline, nil); err != nil { | |
| 1040 | + | if err := d.killSessionWithSnapshotDeadline(s, reason, false, deadline, nil); err != nil { | |
| 1006 | 1041 | checkpointIncomplete = true | |
| 1007 | 1042 | d.log.Error("closing session with unpersisted terminal state", "err", err) | |
| 1008 | 1043 | } | |
| 1009 | - | if !purge && !ephemeral && deadline != nil { | |
| 1044 | + | if !ephemeral && deadline != nil { | |
| 1010 | 1045 | select { | |
| 1011 | 1046 | case <-deadline.Done(): | |
| 1012 | 1047 | checkpointIncomplete = true | |
| @@ -1291,15 +1326,21 @@ func (d *Daemon) handleList(tr ports.ServerConnection) { | |||
| 1291 | 1326 | } | |
| 1292 | 1327 | ||
| 1293 | 1328 | // handleKill terminates the requested live session or stopped named session, | |
| 1294 | - | // or all sessions, and closes the control connection; the resulting EOF is the | |
| 1295 | - | // client's success signal. | |
| 1329 | + | // purges every session (KillAll), or stops the daemon (KillDaemon), and closes | |
| 1330 | + | // the control connection; the resulting EOF is the client's success signal. | |
| 1331 | + | // KillAll leaves the daemon active; KillDaemon is the explicit stop. | |
| 1296 | 1332 | func (d *Daemon) handleKill(tr ports.ServerConnection, request protocol.Kill) { | |
| 1297 | 1333 | defer func() { _ = tr.Close() }() | |
| 1298 | 1334 | ||
| 1299 | 1335 | switch request.Scope { | |
| 1300 | 1336 | case protocol.KillAll: | |
| 1301 | - | if d.purgeAll(protocol.ReasonSessionKilled) { | |
| 1302 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrInternal, "one or more sessions could not be deleted")) | |
| 1337 | + | result := d.purgeAllSessions(protocol.ReasonSessionKilled) | |
| 1338 | + | if result.Superseded() { | |
| 1339 | + | d.logControlSendFailure("kill all superseded response", d.boundedControlSend(tr, serverError(protocol.ErrServerShutdown, "daemon is shutting down"))) | |
| 1340 | + | return | |
| 1341 | + | } | |
| 1342 | + | if result.Failed() { | |
| 1343 | + | d.logControlSendFailure("kill all failure response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, result.summary()))) | |
| 1303 | 1344 | } | |
| 1304 | 1345 | return | |
| 1305 | 1346 | case protocol.KillDaemon: | |
| @@ -1307,7 +1348,7 @@ func (d *Daemon) handleKill(tr ports.ServerConnection, request protocol.Kill) { | |||
| 1307 | 1348 | return | |
| 1308 | 1349 | case protocol.KillSession: | |
| 1309 | 1350 | default: | |
| 1310 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrInternal, "invalid kill scope")) | |
| 1351 | + | d.logControlSendFailure("invalid kill scope response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, "invalid kill scope"))) | |
| 1311 | 1352 | return | |
| 1312 | 1353 | } | |
| 1313 | 1354 | ||
| @@ -1320,7 +1361,7 @@ func (d *Daemon) handleKill(tr ports.ServerConnection, request protocol.Kill) { | |||
| 1320 | 1361 | // deletion order as live and offline purges. | |
| 1321 | 1362 | if err := d.retryStoppedPurge(request.Name); err != nil { | |
| 1322 | 1363 | d.log.Warn("deleting stopped session failed", "err", err, "session", request.Name) | |
| 1323 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrInternal, "deleting stopped session failed")) | |
| 1364 | + | d.logControlSendFailure("stopped delete failure response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, "deleting stopped session failed"))) | |
| 1324 | 1365 | } | |
| 1325 | 1366 | return | |
| 1326 | 1367 | } | |
| @@ -1328,11 +1369,18 @@ func (d *Daemon) handleKill(tr ports.ServerConnection, request protocol.Kill) { | |||
| 1328 | 1369 | d.mu.Unlock() | |
| 1329 | 1370 | ||
| 1330 | 1371 | if target == nil { | |
| 1331 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrNoSuchSession, "no such session: "+request.Name)) | |
| 1372 | + | d.logControlSendFailure("no such session response", d.boundedControlSend(tr, serverError(protocol.ErrNoSuchSession, "no such session: "+request.Name))) | |
| 1332 | 1373 | return | |
| 1333 | 1374 | } | |
| 1334 | 1375 | if err := d.killSession(target, protocol.ReasonSessionKilled, true); err != nil { | |
| 1335 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrInternal, "deleting persisted session failed")) | |
| 1376 | + | if errors.Is(err, errSessionKillParticipantsChanged) { | |
| 1377 | + | // The exact session was not removed; a concurrent detach or token | |
| 1378 | + | // resume invalidated the teardown snapshot. Report the retryable | |
| 1379 | + | // outcome instead of a permanent delete failure. | |
| 1380 | + | d.logControlSendFailure("session delete retry response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, sessionKillRetryMessage))) | |
| 1381 | + | return | |
| 1382 | + | } | |
| 1383 | + | d.logControlSendFailure("session delete failure response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, "deleting persisted session failed"))) | |
| 1336 | 1384 | } | |
| 1337 | 1385 | } | |
| 1338 | 1386 | ||
| @@ -1676,7 +1724,15 @@ func (d *Daemon) routeWithContext(ctx context.Context, h protocol.Hello, tr port | |||
| 1676 | 1724 | if h.ExactTarget != nil && h.ResumeToken == 0 && h.Name != h.ExactTarget.SessionName { | |
| 1677 | 1725 | return nil, nil, &protoErr{protocol.ErrNoSuchSession, "exact session target name mismatch"} | |
| 1678 | 1726 | } | |
| 1727 | + | // A remote exact-target handoff is an admitted transition like a local | |
| 1728 | + | // create/restore: it reserves purge admission before dispatch, so a Hello | |
| 1729 | + | // arriving during a KillAll waits on the gate exactly as a local route does | |
| 1730 | + | // instead of being rejected outright. | |
| 1679 | 1731 | if h.RemoteTarget != nil && h.ResumeToken == 0 { | |
| 1732 | + | if err := d.acquirePurgeAdmission(ctx); err != nil { | |
| 1733 | + | return nil, nil, err | |
| 1734 | + | } | |
| 1735 | + | defer d.releasePurgeAdmission() | |
| 1680 | 1736 | return d.routeRemoteTargetWithContext(ctx, h, tr) | |
| 1681 | 1737 | } | |
| 1682 | 1738 | if h.ExactTarget != nil && h.ResumeToken == 0 { | |
| @@ -1731,11 +1787,20 @@ func (d *Daemon) routeWithContext(ctx context.Context, h protocol.Hello, tr port | |||
| 1731 | 1787 | h.Intent == protocol.IntentEphemeral { | |
| 1732 | 1788 | return nil, nil, &protoErr{protocol.ErrNoSuchTarget, "daemon-owned environment requires an exact remote target"} | |
| 1733 | 1789 | } | |
| 1790 | + | // KillAll admission gate: a purge owns the exact lifecycle set it captured, | |
| 1791 | + | // so a Hello arriving during it waits here and is then admitted as a create | |
| 1792 | + | // or restore after the purge, never inserted into the set being removed. | |
| 1793 | + | // This is transient and re-opened by the purge; it is not daemon closing. | |
| 1794 | + | if err := d.acquirePurgeAdmission(ctx); err != nil { | |
| 1795 | + | return nil, nil, err | |
| 1796 | + | } | |
| 1797 | + | defer d.releasePurgeAdmission() | |
| 1734 | 1798 | d.mu.Lock() | |
| 1735 | - | // Shutdown/create interlock: once shutdown has begun (last session removed, | |
| 1736 | - | // or shutdownAll started) no new session may be created and no attach may | |
| 1799 | + | // Shutdown/create interlock: once an explicit shutdown has begun | |
| 1800 | + | // (shutdownAll started) no new session may be created and no attach may | |
| 1737 | 1801 | // proceed — the teardown snapshot has already been (or is being) taken, so | |
| 1738 | - | // anything inserted now would leak its PTY and hang Serve. | |
| 1802 | + | // anything inserted now would leak its PTY and hang Serve. An empty registry | |
| 1803 | + | // alone is not shutdown, and a KillAll purge is gated by admission above. | |
| 1739 | 1804 | if d.closing { | |
| 1740 | 1805 | d.mu.Unlock() | |
| 1741 | 1806 | return nil, nil, &protoErr{protocol.ErrServerShutdown, "daemon is shutting down"} | |
| @@ -91,11 +91,13 @@ type Daemon struct { | |||||
| 91 | lastAllocatedCreatedAt int64 | 91 | lastAllocatedCreatedAt int64 | ||
| 92 | mruSeq atomic.Uint64 | 92 | mruSeq atomic.Uint64 | ||
| 93 | creationRequestSeq atomic.Uint64 | 93 | creationRequestSeq atomic.Uint64 | ||
| 94 | - | // closing marks that shutdown has irreversibly begun. It is set under mu, | 94 | + | // closing marks that explicit daemon shutdown (KillDaemon/process |
| 95 | - | // atomically with the event that makes shutdown inevitable (the registry | 95 | + | // cancellation) has irreversibly begun. It is set under mu when |
| 96 | - | // emptying in killSession, or shutdownAll starting), and checked by route | 96 | + | // shutdownAll starts, and checked by route under the same mutex — so a Hello |
| 97 | - | // under the same mutex — so a Hello racing shutdown can never insert a new | 97 | + | // racing shutdown can never insert a new session that nobody would tear |
| 98 | - | // session that nobody would tear down. | 98 | + | // down. Neither an empty registry (final session removal) nor a KillAll |
| 99 | + | // purge sets it: the daemon survives empty→occupied→empty and survives a | |||
| 100 | + | // purge, and only an explicit stop/process cancellation ends it. | |||
| 99 | closing bool | 101 | closing bool | ||
| 100 | 102 | ||||
| 101 | // moveLifecycleMu is the daemon-level admission gate for transferable | 103 | // moveLifecycleMu is the daemon-level admission gate for transferable | ||
| @@ -110,6 +112,28 @@ type Daemon struct { | |||||
| 110 | paneProcessCtx context.Context | 112 | paneProcessCtx context.Context | ||
| 111 | paneProcessCancel context.CancelFunc | 113 | paneProcessCancel context.CancelFunc | ||
| 112 | 114 | ||||
| 115 | + | // purgeAllMu grants one daemon-wide purge (KillAll) owner at a time so two | |||
| 116 | + | // concurrent kill-all requests never interleave their exact lifecycle sets. | |||
| 117 | + | // It is not an admission gate; see purgeAdmissionClosing below. | |||
| 118 | + | purgeAllMu sync.Mutex | |||
| 119 | + | // purgeAdmissionClosing is the transient KillAll admission gate. Unlike | |||
| 120 | + | // closing and moveLifecycleClosing it is re-openable: purgeAllSessions sets | |||
| 121 | + | // it under mu, removes exactly the live and stopped/broken/durable records | |||
| 122 | + | // captured at that instant, then clears it so the daemon keeps serving. | |||
| 123 | + | // Creation and restoration wait on the changed channel (a blocked writer | |||
| 124 | + | // keeps new admissions out), while moves and internal creations reject, so | |||
| 125 | + | // no transition can be inserted into the purge's exact lifecycle set. | |||
| 126 | + | // purgeAdmissionActive counts admitted transitions the purge must drain. | |||
| 127 | + | // purgeEpoch increments once at each purge that acquires the gate (after the | |||
| 128 | + | // drain) so a restoration worker blocked on admission can detect a purge | |||
| 129 | + | // that completed while it waited and must not resurrect the records captured | |||
| 130 | + | // before that purge. | |||
| 131 | + | // All three fields are guarded by mu. | |||
| 132 | + | purgeAdmissionClosing bool | |||
| 133 | + | purgeAdmissionActive uint | |||
| 134 | + | purgeAdmissionChanged chan struct{} | |||
| 135 | + | purgeEpoch uint64 | |||
| 136 | + | ||||
| 113 | // notifies holds one completion channel per in-flight async Detached | 137 | // notifies holds one completion channel per in-flight async Detached | ||
| 114 | // notification (guarded by mu, pruned on insert). Channels rather than a | 138 | // notification (guarded by mu, pruned on insert). Channels rather than a | ||
| 115 | // WaitGroup: notifications are spawned from arbitrary goroutines while | 139 | // WaitGroup: notifications are spawned from arbitrary goroutines while | ||
| @@ -180,6 +204,21 @@ type Daemon struct { | |||||
| 180 | // afterAttachmentEffectsFrozen observes the lock-free boundary after all affected | 204 | // afterAttachmentEffectsFrozen observes the lock-free boundary after all affected | ||
| 181 | // attachment gates are frozen and drained, before architecture publication. | 205 | // attachment gates are frozen and drained, before architecture publication. | ||
| 182 | afterAttachmentEffectsFrozen func() | 206 | afterAttachmentEffectsFrozen func() | ||
| 207 | + | // afterPurgeAdmission is a deterministic test seam that runs after a KillAll | |||
| 208 | + | // purge has closed purge admission and taken the closing fence, while it | |||
| 209 | + | // still owns purgeAllMu. It lets tests observe purge serialization. | |||
| 210 | + | afterPurgeAdmission func() | |||
| 211 | + | // beforePurgeLiveSessionKill is a deterministic test seam that runs after a | |||
| 212 | + | // KillAll purge's per-unit shutdown precheck and before that live unit | |||
| 213 | + | // attempts teardown ownership. Tests use it to let an explicit stop win that | |||
| 214 | + | // exact window, proving a stop-owned unit is never reported as a success. | |||
| 215 | + | beforePurgeLiveSessionKill func(*session) | |||
| 216 | + | // afterSessionKillQuarantine is a deterministic test seam that runs while a | |||
| 217 | + | // teardown owns the session and its snapshot coordinator is quarantined, but | |||
| 218 | + | // before the pre-publication revalidation. Tests use it to change the | |||
| 219 | + | // participant set in that window and prove a post-quarantine abort rolls the | |||
| 220 | + | // quarantine back so a surviving session keeps checkpointing. | |||
| 221 | + | afterSessionKillQuarantine func(*session) | |||
| 183 | // beforeAttachmentTransitionIdentityAdmission is a deterministic test seam | 222 | // beforeAttachmentTransitionIdentityAdmission is a deterministic test seam | ||
| 184 | // after a ready transition published its capability and before the committed | 223 | // after a ready transition published its capability and before the committed | ||
| 185 | // route identity is admitted, so tests can supersede that exact capability in | 224 | // route identity is admitted, so tests can supersede that exact capability in | ||
| @@ -348,6 +387,10 @@ type Daemon struct { | |||||
| 348 | hardCtx context.Context | 387 | hardCtx context.Context | ||
| 349 | hardCancel context.CancelFunc | 388 | hardCancel context.CancelFunc | ||
| 350 | 389 | ||||
| 390 | + | // done is closed exactly once by an explicit daemon stop (shutdownAll) to | |||
| 391 | + | // break Serve's accept loop. It is deliberately not closed by final session | |||
| 392 | + | // removal or by a KillAll purge: an empty registry and a purged registry | |||
| 393 | + | // both keep the daemon serving. | |||
| 351 | done chan struct{} | 394 | done chan struct{} | ||
| 352 | doneOnce sync.Once | 395 | doneOnce sync.Once | ||
| 353 | 396 | ||||
| @@ -447,6 +490,10 @@ func WithRuntimeObserver(observer ports.SerializedRuntimeObserver) Option { | |||||
| 447 | // WithRemoteMonitor installs the snapshot-only remote directory and the | 490 | // WithRemoteMonitor installs the snapshot-only remote directory and the | ||
| 448 | // monitor runner. Serve starts the runner without waiting for registry, | 491 | // monitor runner. Serve starts the runner without waiting for registry, | ||
| 449 | // cache or runtime readiness and stops it with its own context. | 492 | // cache or runtime readiness and stops it with its own context. | ||
| 493 | + | // | |||
| 494 | + | // The daemon-side monitor stays intentionally active in this state: it is part | |||
| 495 | + | // of the coordinated P7 removal set, so no polling gate, feature switch, or | |||
| 496 | + | // early shutdown path is added here (GO-001, deferred to P7). | |||
| 450 | func WithRemoteMonitor(directory ports.RemoteDirectory, run func(context.Context) error) Option { | 497 | func WithRemoteMonitor(directory ports.RemoteDirectory, run func(context.Context) error) Option { | ||
| 451 | return func(d *Daemon) { | 498 | return func(d *Daemon) { | ||
| 452 | d.remoteDirectory = directory | 499 | d.remoteDirectory = directory | ||
| @@ -792,8 +839,12 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||||
| 792 | const remoteMonitorShutdownJoin = 3 * time.Second | 839 | const remoteMonitorShutdownJoin = 3 * time.Second | ||
| 793 | 840 | ||||
| 794 | // Serve runs the accept loop over l, owning it for the loop's lifetime. It | 841 | // Serve runs the accept loop over l, owning it for the loop's lifetime. It | ||
| 795 | - | // returns when the last session is removed or ctx is cancelled; on the latter | 842 | + | // returns only on ctx cancellation or an explicit daemon shutdown request |
| 796 | - | // path attached clients are detached with ReasonServerShutdown. | 843 | + | // (KillDaemon), never when the session registry drains to empty: final |
| 844 | + | // session removal is independent of daemon shutdown so the daemon survives | |||
| 845 | + | // empty→occupied→empty and can accept fresh sessions again. A KillAll purge | |||
| 846 | + | // leaves Serve running too. On the ctx path | |||
| 847 | + | // attached clients are detached with ReasonServerShutdown. | |||
| 797 | func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | 848 | func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | ||
| 798 | d.serveCtx, d.serveCancel = context.WithCancel(ctx) | 849 | d.serveCtx, d.serveCancel = context.WithCancel(ctx) | ||
| 799 | defer d.serveCancel() | 850 | defer d.serveCancel() | ||
| @@ -866,8 +917,9 @@ func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | |||||
| 866 | } else { | 917 | } else { | ||
| 867 | d.closeRestoreDone() | 918 | d.closeRestoreDone() | ||
| 868 | } | 919 | } | ||
| 869 | - | // Break the accept loop when either the parent context is cancelled or the | 920 | + | // Break the accept loop only on an explicit shutdown: the parent context |
| 870 | - | // registry drains to empty: both close the listener, which fails Accept. | 921 | + | // being cancelled or an explicit daemon stop request (d.done). An empty |
| 922 | + | // registry never closes the listener, so the daemon keeps accepting. | |||
| 871 | go func() { | 923 | go func() { | ||
| 872 | select { | 924 | select { | ||
| 873 | case <-d.serveCtx.Done(): | 925 | case <-d.serveCtx.Done(): | ||
| @@ -908,7 +960,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | |||||
| 908 | snapshotDeadline = newSnapshotShutdownDeadline(d.clock) | 960 | snapshotDeadline = newSnapshotShutdownDeadline(d.clock) | ||
| 909 | defer snapshotDeadline.stop() | 961 | defer snapshotDeadline.stop() | ||
| 910 | } | 962 | } | ||
| 911 | - | d.terminateAllWithSnapshotDeadline(protocol.ReasonServerShutdown, false, snapshotDeadline) | 963 | + | d.terminateAllForShutdown(protocol.ReasonServerShutdown, snapshotDeadline) |
| 912 | d.waitNotifies() | 964 | d.waitNotifies() | ||
| 913 | d.hardCancel() | 965 | d.hardCancel() | ||
| 914 | d.serveCancel() | 966 | d.serveCancel() | ||
| @@ -928,7 +980,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | |||||
| 928 | d.persistShutdownSnapshotFailure(name, context.DeadlineExceeded) | 980 | d.persistShutdownSnapshotFailure(name, context.DeadlineExceeded) | ||
| 929 | } | 981 | } | ||
| 930 | d.WaitDurableWriters() | 982 | d.WaitDurableWriters() | ||
| 931 | - | d.terminateAllWithSnapshotDeadline(protocol.ReasonServerShutdown, false, snapshotDeadline) | 983 | + | d.terminateAllForShutdown(protocol.ReasonServerShutdown, snapshotDeadline) |
| 932 | d.waitSessionWorkersWithSnapshotDeadline(snapshotDeadline) | 984 | d.waitSessionWorkersWithSnapshotDeadline(snapshotDeadline) | ||
| 933 | d.waitNotifies() | 985 | d.waitNotifies() | ||
| 934 | if err := d.flushCatalogue(); err != nil { | 986 | if err := d.flushCatalogue(); err != nil { | ||
| @@ -942,58 +994,41 @@ func (d *Daemon) Serve(ctx context.Context, l ports.ServerListener) error { | |||||
| 942 | return nil | 994 | return nil | ||
| 943 | } | 995 | } | ||
| 944 | 996 | ||||
| 945 | - | // shutdownAll marks the daemon closing and kills every live session. Setting | 997 | + | // shutdownAll is the explicit daemon-stop operation (wire KillDaemon). It is |
| 946 | - | // closing under the same lock as the snapshot guarantees no session can be | 998 | + | // the only caller that irreversibly ends the daemon: it closes move admission, |
| 947 | - | // inserted after the snapshot: route rejects once closing is set, and both run | 999 | + | // marks closing, cancels the daemon-wide pane process context, preserves every |
| 948 | - | // under d.mu. killSession (which relocks) runs after the lock is released. | 1000 | + | // live session as stopped durable authority, and closes done so Serve returns. |
| 1001 | + | // KillAll never reaches this path. Setting closing under the same lock as the | |||
| 1002 | + | // snapshot guarantees no session can be inserted after the snapshot: route | |||
| 1003 | + | // rejects once closing is set, and both run under d.mu. killSession (which | |||
| 1004 | + | // relocks) runs after the lock is released. | |||
| 949 | func (d *Daemon) shutdownAll(reason uint8) (checkpointIncomplete bool) { | 1005 | func (d *Daemon) shutdownAll(reason uint8) (checkpointIncomplete bool) { | ||
| 950 | - | return d.terminateAllWithSnapshotDeadline(reason, false, nil) | 1006 | + | return d.terminateAllForShutdown(reason, nil) |
| 951 | - | } | |||
| 952 | - | ||||
| 953 | - | func (d *Daemon) purgeAll(reason uint8) (checkpointIncomplete bool) { | |||
| 954 | - | return d.terminateAllWithSnapshotDeadline(reason, true, nil) | |||
| 955 | } | 1007 | } | ||
| 956 | 1008 | ||||
| 957 | - | func (d *Daemon) terminateAllWithSnapshotDeadline(reason uint8, purge bool, deadline *snapshotShutdownDeadline) (checkpointIncomplete bool) { | 1009 | + | // terminateAllForShutdown performs the irreversible daemon teardown shared by |
| 1010 | + | // shutdownAll and Serve's exit path. It is separate from purgeAllSessions, | |||
| 1011 | + | // which leaves every daemon-wide shutdown fact untouched. | |||
| 1012 | + | func (d *Daemon) terminateAllForShutdown(reason uint8, deadline *snapshotShutdownDeadline) (checkpointIncomplete bool) { | |||
| 958 | d.closeMoveLifecycles() | 1013 | d.closeMoveLifecycles() | ||
| 959 | d.mu.Lock() | 1014 | d.mu.Lock() | ||
| 960 | d.closing = true | 1015 | d.closing = true | ||
| 1016 | + | // Wake any KillAll waiting on purge admission so it defers to this stop | |||
| 1017 | + | // instead of holding the gate until its bounded drain deadline expires. | |||
| 1018 | + | d.signalPurgeAdmissionChangedLocked() | |||
| 961 | d.purgeAllParkingLocked() | 1019 | d.purgeAllParkingLocked() | ||
| 962 | parkedRetirements := d.purgeAllParkedLocked() | 1020 | parkedRetirements := d.purgeAllParkedLocked() | ||
| 963 | d.purgeAllSuspendedLocked() | 1021 | d.purgeAllSuspendedLocked() | ||
| 964 | snapshot := d.sessionsSnapshotLocked() | 1022 | snapshot := d.sessionsSnapshotLocked() | ||
| 965 | - | stoppedNames := make([]string, 0, len(d.inactive)) | |||
| 966 | - | if purge { | |||
| 967 | - | for name := range d.inactive { | |||
| 968 | - | stoppedNames = append(stoppedNames, name) | |||
| 969 | - | } | |||
| 970 | - | sort.Strings(stoppedNames) | |||
| 971 | - | } | |||
| 972 | - | empty := len(snapshot) == 0 | |||
| 973 | d.mu.Unlock() | 1023 | d.mu.Unlock() | ||
| 974 | d.finishParkedAttachmentRetirements(parkedRetirements) | 1024 | d.finishParkedAttachmentRetirements(parkedRetirements) | ||
| 975 | - | if !purge { | 1025 | + | // Publish preservation policy for the complete registry snapshot before |
| 976 | - | // Publish preservation policy for the complete registry snapshot before | 1026 | + | // cancelling any PTY. Cancellation-driven EOF is allowed to own teardown, |
| 977 | - | // cancelling any PTY. Cancellation-driven EOF is allowed to own teardown, | 1027 | + | // but its disposition must remain daemon shutdown rather than session purge. |
| 978 | - | // but its disposition must remain daemon shutdown rather than session purge. | 1028 | + | for _, s := range snapshot { |
| 979 | - | for _, s := range snapshot { | 1029 | + | s.reserveShutdownTeardown() |
| 980 | - | s.reserveShutdownTeardown() | |||
| 981 | - | } | |||
| 982 | - | } | |||
| 983 | - | d.log.Info("session termination begin", "reason", reason, "purge", purge, "live_sessions", len(snapshot), "stopped_sessions", len(stoppedNames)) | |||
| 984 | - | // Purge stopped authority before signaling Serve to exit. Once the final | |||
| 985 | - | // live registry entry closes, Serve cancels its context; deleting stopped | |||
| 986 | - | // records afterward would be interrupted and retain broken catalogue entries. | |||
| 987 | - | for _, name := range stoppedNames { | |||
| 988 | - | if err := d.retryStoppedPurge(name); err != nil { | |||
| 989 | - | checkpointIncomplete = true | |||
| 990 | - | d.log.Error("deleting stopped session during kill all failed", "err", err, "session", name) | |||
| 991 | - | } | |||
| 992 | - | } | |||
| 993 | - | if empty { | |||
| 994 | - | d.doneOnce.Do(func() { close(d.done) }) | |||
| 995 | - | return checkpointIncomplete | |||
| 996 | } | 1030 | } | ||
| 1031 | + | d.log.Info("session termination begin", "reason", reason, "live_sessions", len(snapshot)) | |||
| 997 | for _, s := range snapshot { | 1032 | for _, s := range snapshot { | ||
| 998 | // Cancellation and PTY closure must not wait behind a teardown owner that | 1033 | // Cancellation and PTY closure must not wait behind a teardown owner that | ||
| 999 | // is blocked in snapshot publication or purge work. | 1034 | // is blocked in snapshot publication or purge work. | ||
| @@ -1002,11 +1037,11 @@ func (d *Daemon) terminateAllWithSnapshotDeadline(reason uint8, purge bool, dead | |||||
| 1002 | ephemeral := s.ephemeral | 1037 | ephemeral := s.ephemeral | ||
| 1003 | s.mu.Unlock() | 1038 | s.mu.Unlock() | ||
| 1004 | s.stopInMemoryLifecycle() | 1039 | s.stopInMemoryLifecycle() | ||
| 1005 | - | if err := d.killSessionWithSnapshotDeadline(s, reason, purge, deadline, nil); err != nil { | 1040 | + | if err := d.killSessionWithSnapshotDeadline(s, reason, false, deadline, nil); err != nil { |
| 1006 | checkpointIncomplete = true | 1041 | checkpointIncomplete = true | ||
| 1007 | d.log.Error("closing session with unpersisted terminal state", "err", err) | 1042 | d.log.Error("closing session with unpersisted terminal state", "err", err) | ||
| 1008 | } | 1043 | } | ||
| 1009 | - | if !purge && !ephemeral && deadline != nil { | 1044 | + | if !ephemeral && deadline != nil { |
| 1010 | select { | 1045 | select { | ||
| 1011 | case <-deadline.Done(): | 1046 | case <-deadline.Done(): | ||
| 1012 | checkpointIncomplete = true | 1047 | checkpointIncomplete = true | ||
| @@ -1291,15 +1326,21 @@ func (d *Daemon) handleList(tr ports.ServerConnection) { | |||||
| 1291 | } | 1326 | } | ||
| 1292 | 1327 | ||||
| 1293 | // handleKill terminates the requested live session or stopped named session, | 1328 | // handleKill terminates the requested live session or stopped named session, | ||
| 1294 | - | // or all sessions, and closes the control connection; the resulting EOF is the | 1329 | + | // purges every session (KillAll), or stops the daemon (KillDaemon), and closes |
| 1295 | - | // client's success signal. | 1330 | + | // the control connection; the resulting EOF is the client's success signal. |
| 1331 | + | // KillAll leaves the daemon active; KillDaemon is the explicit stop. | |||
| 1296 | func (d *Daemon) handleKill(tr ports.ServerConnection, request protocol.Kill) { | 1332 | func (d *Daemon) handleKill(tr ports.ServerConnection, request protocol.Kill) { | ||
| 1297 | defer func() { _ = tr.Close() }() | 1333 | defer func() { _ = tr.Close() }() | ||
| 1298 | 1334 | ||||
| 1299 | switch request.Scope { | 1335 | switch request.Scope { | ||
| 1300 | case protocol.KillAll: | 1336 | case protocol.KillAll: | ||
| 1301 | - | if d.purgeAll(protocol.ReasonSessionKilled) { | 1337 | + | result := d.purgeAllSessions(protocol.ReasonSessionKilled) |
| 1302 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrInternal, "one or more sessions could not be deleted")) | 1338 | + | if result.Superseded() { |
| 1339 | + | d.logControlSendFailure("kill all superseded response", d.boundedControlSend(tr, serverError(protocol.ErrServerShutdown, "daemon is shutting down"))) | |||
| 1340 | + | return | |||
| 1341 | + | } | |||
| 1342 | + | if result.Failed() { | |||
| 1343 | + | d.logControlSendFailure("kill all failure response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, result.summary()))) | |||
| 1303 | } | 1344 | } | ||
| 1304 | return | 1345 | return | ||
| 1305 | case protocol.KillDaemon: | 1346 | case protocol.KillDaemon: | ||
| @@ -1307,7 +1348,7 @@ func (d *Daemon) handleKill(tr ports.ServerConnection, request protocol.Kill) { | |||||
| 1307 | return | 1348 | return | ||
| 1308 | case protocol.KillSession: | 1349 | case protocol.KillSession: | ||
| 1309 | default: | 1350 | default: | ||
| 1310 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrInternal, "invalid kill scope")) | 1351 | + | d.logControlSendFailure("invalid kill scope response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, "invalid kill scope"))) |
| 1311 | return | 1352 | return | ||
| 1312 | } | 1353 | } | ||
| 1313 | 1354 | ||||
| @@ -1320,7 +1361,7 @@ func (d *Daemon) handleKill(tr ports.ServerConnection, request protocol.Kill) { | |||||
| 1320 | // deletion order as live and offline purges. | 1361 | // deletion order as live and offline purges. | ||
| 1321 | if err := d.retryStoppedPurge(request.Name); err != nil { | 1362 | if err := d.retryStoppedPurge(request.Name); err != nil { | ||
| 1322 | d.log.Warn("deleting stopped session failed", "err", err, "session", request.Name) | 1363 | d.log.Warn("deleting stopped session failed", "err", err, "session", request.Name) | ||
| 1323 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrInternal, "deleting stopped session failed")) | 1364 | + | d.logControlSendFailure("stopped delete failure response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, "deleting stopped session failed"))) |
| 1324 | } | 1365 | } | ||
| 1325 | return | 1366 | return | ||
| 1326 | } | 1367 | } | ||
| @@ -1328,11 +1369,18 @@ func (d *Daemon) handleKill(tr ports.ServerConnection, request protocol.Kill) { | |||||
| 1328 | d.mu.Unlock() | 1369 | d.mu.Unlock() | ||
| 1329 | 1370 | ||||
| 1330 | if target == nil { | 1371 | if target == nil { | ||
| 1331 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrNoSuchSession, "no such session: "+request.Name)) | 1372 | + | d.logControlSendFailure("no such session response", d.boundedControlSend(tr, serverError(protocol.ErrNoSuchSession, "no such session: "+request.Name))) |
| 1332 | return | 1373 | return | ||
| 1333 | } | 1374 | } | ||
| 1334 | if err := d.killSession(target, protocol.ReasonSessionKilled, true); err != nil { | 1375 | if err := d.killSession(target, protocol.ReasonSessionKilled, true); err != nil { | ||
| 1335 | - | _ = d.boundedControlSend(tr, serverError(protocol.ErrInternal, "deleting persisted session failed")) | 1376 | + | if errors.Is(err, errSessionKillParticipantsChanged) { |
| 1377 | + | // The exact session was not removed; a concurrent detach or token | |||
| 1378 | + | // resume invalidated the teardown snapshot. Report the retryable | |||
| 1379 | + | // outcome instead of a permanent delete failure. | |||
| 1380 | + | d.logControlSendFailure("session delete retry response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, sessionKillRetryMessage))) | |||
| 1381 | + | return | |||
| 1382 | + | } | |||
| 1383 | + | d.logControlSendFailure("session delete failure response", d.boundedControlSend(tr, serverError(protocol.ErrInternal, "deleting persisted session failed"))) | |||
| 1336 | } | 1384 | } | ||
| 1337 | } | 1385 | } | ||
| 1338 | 1386 | ||||
| @@ -1676,7 +1724,15 @@ func (d *Daemon) routeWithContext(ctx context.Context, h protocol.Hello, tr port | |||||
| 1676 | if h.ExactTarget != nil && h.ResumeToken == 0 && h.Name != h.ExactTarget.SessionName { | 1724 | if h.ExactTarget != nil && h.ResumeToken == 0 && h.Name != h.ExactTarget.SessionName { | ||
| 1677 | return nil, nil, &protoErr{protocol.ErrNoSuchSession, "exact session target name mismatch"} | 1725 | return nil, nil, &protoErr{protocol.ErrNoSuchSession, "exact session target name mismatch"} | ||
| 1678 | } | 1726 | } | ||
| 1727 | + | // A remote exact-target handoff is an admitted transition like a local | |||
| 1728 | + | // create/restore: it reserves purge admission before dispatch, so a Hello | |||
| 1729 | + | // arriving during a KillAll waits on the gate exactly as a local route does | |||
| 1730 | + | // instead of being rejected outright. | |||
| 1679 | if h.RemoteTarget != nil && h.ResumeToken == 0 { | 1731 | if h.RemoteTarget != nil && h.ResumeToken == 0 { | ||
| 1732 | + | if err := d.acquirePurgeAdmission(ctx); err != nil { | |||
| 1733 | + | return nil, nil, err | |||
| 1734 | + | } | |||
| 1735 | + | defer d.releasePurgeAdmission() | |||
| 1680 | return d.routeRemoteTargetWithContext(ctx, h, tr) | 1736 | return d.routeRemoteTargetWithContext(ctx, h, tr) | ||
| 1681 | } | 1737 | } | ||
| 1682 | if h.ExactTarget != nil && h.ResumeToken == 0 { | 1738 | if h.ExactTarget != nil && h.ResumeToken == 0 { | ||
| @@ -1731,11 +1787,20 @@ func (d *Daemon) routeWithContext(ctx context.Context, h protocol.Hello, tr port | |||||
| 1731 | h.Intent == protocol.IntentEphemeral { | 1787 | h.Intent == protocol.IntentEphemeral { | ||
| 1732 | return nil, nil, &protoErr{protocol.ErrNoSuchTarget, "daemon-owned environment requires an exact remote target"} | 1788 | return nil, nil, &protoErr{protocol.ErrNoSuchTarget, "daemon-owned environment requires an exact remote target"} | ||
| 1733 | } | 1789 | } | ||
| 1790 | + | // KillAll admission gate: a purge owns the exact lifecycle set it captured, | |||
| 1791 | + | // so a Hello arriving during it waits here and is then admitted as a create | |||
| 1792 | + | // or restore after the purge, never inserted into the set being removed. | |||
| 1793 | + | // This is transient and re-opened by the purge; it is not daemon closing. | |||
| 1794 | + | if err := d.acquirePurgeAdmission(ctx); err != nil { | |||
| 1795 | + | return nil, nil, err | |||
| 1796 | + | } | |||
| 1797 | + | defer d.releasePurgeAdmission() | |||
| 1734 | d.mu.Lock() | 1798 | d.mu.Lock() | ||
| 1735 | - | // Shutdown/create interlock: once shutdown has begun (last session removed, | 1799 | + | // Shutdown/create interlock: once an explicit shutdown has begun |
| 1736 | - | // or shutdownAll started) no new session may be created and no attach may | 1800 | + | // (shutdownAll started) no new session may be created and no attach may |
| 1737 | // proceed — the teardown snapshot has already been (or is being) taken, so | 1801 | // proceed — the teardown snapshot has already been (or is being) taken, so | ||
| 1738 | - | // anything inserted now would leak its PTY and hang Serve. | 1802 | + | // anything inserted now would leak its PTY and hang Serve. An empty registry |
| 1803 | + | // alone is not shutdown, and a KillAll purge is gated by admission above. | |||
| 1739 | if d.closing { | 1804 | if d.closing { | ||
| 1740 | d.mu.Unlock() | 1805 | d.mu.Unlock() | ||
| 1741 | return nil, nil, &protoErr{protocol.ErrServerShutdown, "daemon is shutting down"} | 1806 | return nil, nil, &protoErr{protocol.ErrServerShutdown, "daemon is shutting down"} | ||
M internal/usecase/daemon/durable_recovery_test.go +18 -18
| @@ -275,10 +275,10 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) { | |||
| 275 | 275 | ||
| 276 | 276 | func TestCatalogueRestoreIndependent(t *testing.T) { | |
| 277 | 277 | t.Run("empty catalogue restoration", func(t *testing.T) { | |
| 278 | - | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | |
| 278 | + | emptyDaemon, _ := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | |
| 279 | 279 | emptyDone := make(chan struct{}) | |
| 280 | 280 | go func() { | |
| 281 | - | emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue)) | |
| 281 | + | emptyDaemon.restoreIncrementalSnapshots(context.Background()) | |
| 282 | 282 | close(emptyDone) | |
| 283 | 283 | }() | |
| 284 | 284 | select { | |
| @@ -299,7 +299,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||
| 299 | 299 | } | |
| 300 | 300 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | |
| 301 | 301 | ||
| 302 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 302 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 303 | 303 | ||
| 304 | 304 | d.mu.Lock() | |
| 305 | 305 | healthyEntry := d.inactive[healthy.Name] | |
| @@ -335,7 +335,7 @@ func TestCatalogueRestoreResetsOnlyIncompatibleCheckpoint(t *testing.T) { | |||
| 335 | 335 | } | |
| 336 | 336 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{incompatible, healthy}, repository) | |
| 337 | 337 | ||
| 338 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 338 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 339 | 339 | ||
| 340 | 340 | fresh, ok, err := catalogue.Record(incompatible.Name) | |
| 341 | 341 | require.NoError(t, err) | |
| @@ -370,7 +370,7 @@ func TestIncompatibleCheckpointDeleteFailurePublishesFreshStoppedAuthority(t *te | |||
| 370 | 370 | restoreDone := d.inactive[record.Name].restoreDone | |
| 371 | 371 | d.mu.Unlock() | |
| 372 | 372 | ||
| 373 | - | restoreErr := d.restoreRecord(context.Background(), record) | |
| 373 | + | restoreErr := d.restoreRecord(context.Background(), record, d.purgeEpochSnapshot()) | |
| 374 | 374 | require.NoError(t, restoreErr, "post-commit cleanup failure must not fail the authority transition") | |
| 375 | 375 | ||
| 376 | 376 | fresh, ok, err := catalogue.Record(record.Name) | |
| @@ -416,7 +416,7 @@ func TestIncompatibleCheckpointPreCommitFailureReturnsError(t *testing.T) { | |||
| 416 | 416 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 417 | 417 | catalogue.replaceErr = cause | |
| 418 | 418 | ||
| 419 | - | err := d.restoreRecord(context.Background(), record) | |
| 419 | + | err := d.restoreRecord(context.Background(), record, d.purgeEpochSnapshot()) | |
| 420 | 420 | ||
| 421 | 421 | require.ErrorIs(t, err, cause) | |
| 422 | 422 | persisted, ok, recordErr := catalogue.Record(record.Name) | |
| @@ -445,7 +445,7 @@ func TestRestoreLoadFailuresRetainCheckpointWithoutDeletion(t *testing.T) { | |||
| 445 | 445 | } | |
| 446 | 446 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 447 | 447 | ||
| 448 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 448 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 449 | 449 | ||
| 450 | 450 | persisted, ok, err := catalogue.Record(record.Name) | |
| 451 | 451 | require.NoError(t, err) | |
| @@ -467,9 +467,9 @@ func TestCatalogueRestoreAggregateOver4096(t *testing.T) { | |||
| 467 | 467 | records[i] = durableRecoveryRecord(i) | |
| 468 | 468 | repository.generations[records[i].Name] = validGeneration(t, records[i]) | |
| 469 | 469 | } | |
| 470 | - | d, catalogue := newDurableRecoveryDaemon(t, records, repository) | |
| 470 | + | d, _ := newDurableRecoveryDaemon(t, records, repository) | |
| 471 | 471 | ||
| 472 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 472 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 473 | 473 | ||
| 474 | 474 | require.Len(t, d.inactive, count) | |
| 475 | 475 | for _, record := range records { | |
| @@ -481,11 +481,11 @@ func TestCatalogueRestoreAggregateOver4096(t *testing.T) { | |||
| 481 | 481 | func TestCatalogueRestoreSingleSessionOver4096(t *testing.T) { | |
| 482 | 482 | record := durableRecoveryRecord(0) | |
| 483 | 483 | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: validGeneration(t, record)}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | |
| 484 | - | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 484 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 485 | 485 | ||
| 486 | 486 | // The repository may contain any number of stale entries; restoration has no | |
| 487 | 487 | // discovery API and therefore performs exactly one direct catalogue lookup. | |
| 488 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 488 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 489 | 489 | ||
| 490 | 490 | require.Equal(t, protocol.SessionDown, d.inactive[record.Name].state) | |
| 491 | 491 | require.Equal(t, 1, repository.loads[record.Name]) | |
| @@ -498,7 +498,7 @@ func TestCatalogueRestoreIncarnationMismatch(t *testing.T) { | |||
| 498 | 498 | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: generation}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | |
| 499 | 499 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 500 | 500 | ||
| 501 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 501 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 502 | 502 | ||
| 503 | 503 | require.Equal(t, protocol.SessionBroken, d.inactive[record.Name].state) | |
| 504 | 504 | got, _, _ := catalogue.Record(record.Name) | |
| @@ -674,7 +674,7 @@ func TestPersistAndRegisterRestoredSessionRegistrationFailurePreservesStoppedEnt | |||
| 674 | 674 | sess := newSnapshotTestSession(t, record.Name, false, record.Cwd) | |
| 675 | 675 | sess.incarnation = record.IncarnationID | |
| 676 | 676 | ||
| 677 | - | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess) | |
| 677 | + | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess, d.purgeEpochSnapshot()) | |
| 678 | 678 | ||
| 679 | 679 | require.ErrorContains(t, err, "registry rejected") | |
| 680 | 680 | require.False(t, registered) | |
| @@ -744,7 +744,7 @@ func TestPersistAndRegisterRestoredSessionRejectsReplacementIncarnation(t *testi | |||
| 744 | 744 | sess := newSnapshotTestSession(t, record.Name, false, record.Cwd) | |
| 745 | 745 | sess.incarnation = record.IncarnationID | |
| 746 | 746 | ||
| 747 | - | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess) | |
| 747 | + | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess, d.purgeEpochSnapshot()) | |
| 748 | 748 | require.ErrorContains(t, err, "incarnation") | |
| 749 | 749 | require.False(t, registered) | |
| 750 | 750 | require.Empty(t, d.sessions) | |
| @@ -842,7 +842,7 @@ func TestAttachBarrierSurvivesLiveRegistryPublication(t *testing.T) { | |||
| 842 | 842 | d.mu.Unlock() | |
| 843 | 843 | sess := &session{sessionCore: sessionCore{name: record.Name, incarnation: record.IncarnationID}} | |
| 844 | 844 | ||
| 845 | - | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess) | |
| 845 | + | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess, d.purgeEpochSnapshot()) | |
| 846 | 846 | require.NoError(t, err) | |
| 847 | 847 | require.True(t, registered) | |
| 848 | 848 | ||
| @@ -879,7 +879,7 @@ func TestRestoreCancellationTransitionsBeforeAttachCompletion(t *testing.T) { | |||
| 879 | 879 | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | |
| 880 | 880 | restored := make(chan struct{}) | |
| 881 | 881 | go func() { | |
| 882 | - | d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue)) | |
| 882 | + | d.restoreIncrementalSnapshots(restoreCtx) | |
| 883 | 883 | close(restored) | |
| 884 | 884 | }() | |
| 885 | 885 | <-repository.started | |
| @@ -990,9 +990,9 @@ func TestCatalogueRestoreDegradedVisible(t *testing.T) { | |||
| 990 | 990 | record := durableRecoveryRecord(0) | |
| 991 | 991 | record.DegradedReason = "checkpoint validation failed" | |
| 992 | 992 | repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | |
| 993 | - | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 993 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 994 | 994 | ||
| 995 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 995 | + | d.restoreIncrementalSnapshots(context.Background()) | |
| 996 | 996 | ||
| 997 | 997 | require.Equal(t, protocol.SessionBroken, d.inactive[record.Name].state) | |
| 998 | 998 | require.Zero(t, repository.loads[record.Name]) | |
| @@ -275,10 +275,10 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) { | |||||
| 275 | 275 | ||||
| 276 | func TestCatalogueRestoreIndependent(t *testing.T) { | 276 | func TestCatalogueRestoreIndependent(t *testing.T) { | ||
| 277 | t.Run("empty catalogue restoration", func(t *testing.T) { | 277 | t.Run("empty catalogue restoration", func(t *testing.T) { | ||
| 278 | - | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | 278 | + | emptyDaemon, _ := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) |
| 279 | emptyDone := make(chan struct{}) | 279 | emptyDone := make(chan struct{}) | ||
| 280 | go func() { | 280 | go func() { | ||
| 281 | - | emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue)) | 281 | + | emptyDaemon.restoreIncrementalSnapshots(context.Background()) |
| 282 | close(emptyDone) | 282 | close(emptyDone) | ||
| 283 | }() | 283 | }() | ||
| 284 | select { | 284 | select { | ||
| @@ -299,7 +299,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||||
| 299 | } | 299 | } | ||
| 300 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | 300 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | ||
| 301 | 301 | ||||
| 302 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 302 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 303 | 303 | ||||
| 304 | d.mu.Lock() | 304 | d.mu.Lock() | ||
| 305 | healthyEntry := d.inactive[healthy.Name] | 305 | healthyEntry := d.inactive[healthy.Name] | ||
| @@ -335,7 +335,7 @@ func TestCatalogueRestoreResetsOnlyIncompatibleCheckpoint(t *testing.T) { | |||||
| 335 | } | 335 | } | ||
| 336 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{incompatible, healthy}, repository) | 336 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{incompatible, healthy}, repository) | ||
| 337 | 337 | ||||
| 338 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 338 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 339 | 339 | ||||
| 340 | fresh, ok, err := catalogue.Record(incompatible.Name) | 340 | fresh, ok, err := catalogue.Record(incompatible.Name) | ||
| 341 | require.NoError(t, err) | 341 | require.NoError(t, err) | ||
| @@ -370,7 +370,7 @@ func TestIncompatibleCheckpointDeleteFailurePublishesFreshStoppedAuthority(t *te | |||||
| 370 | restoreDone := d.inactive[record.Name].restoreDone | 370 | restoreDone := d.inactive[record.Name].restoreDone | ||
| 371 | d.mu.Unlock() | 371 | d.mu.Unlock() | ||
| 372 | 372 | ||||
| 373 | - | restoreErr := d.restoreRecord(context.Background(), record) | 373 | + | restoreErr := d.restoreRecord(context.Background(), record, d.purgeEpochSnapshot()) |
| 374 | require.NoError(t, restoreErr, "post-commit cleanup failure must not fail the authority transition") | 374 | require.NoError(t, restoreErr, "post-commit cleanup failure must not fail the authority transition") | ||
| 375 | 375 | ||||
| 376 | fresh, ok, err := catalogue.Record(record.Name) | 376 | fresh, ok, err := catalogue.Record(record.Name) | ||
| @@ -416,7 +416,7 @@ func TestIncompatibleCheckpointPreCommitFailureReturnsError(t *testing.T) { | |||||
| 416 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 416 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | ||
| 417 | catalogue.replaceErr = cause | 417 | catalogue.replaceErr = cause | ||
| 418 | 418 | ||||
| 419 | - | err := d.restoreRecord(context.Background(), record) | 419 | + | err := d.restoreRecord(context.Background(), record, d.purgeEpochSnapshot()) |
| 420 | 420 | ||||
| 421 | require.ErrorIs(t, err, cause) | 421 | require.ErrorIs(t, err, cause) | ||
| 422 | persisted, ok, recordErr := catalogue.Record(record.Name) | 422 | persisted, ok, recordErr := catalogue.Record(record.Name) | ||
| @@ -445,7 +445,7 @@ func TestRestoreLoadFailuresRetainCheckpointWithoutDeletion(t *testing.T) { | |||||
| 445 | } | 445 | } | ||
| 446 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 446 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | ||
| 447 | 447 | ||||
| 448 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 448 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 449 | 449 | ||||
| 450 | persisted, ok, err := catalogue.Record(record.Name) | 450 | persisted, ok, err := catalogue.Record(record.Name) | ||
| 451 | require.NoError(t, err) | 451 | require.NoError(t, err) | ||
| @@ -467,9 +467,9 @@ func TestCatalogueRestoreAggregateOver4096(t *testing.T) { | |||||
| 467 | records[i] = durableRecoveryRecord(i) | 467 | records[i] = durableRecoveryRecord(i) | ||
| 468 | repository.generations[records[i].Name] = validGeneration(t, records[i]) | 468 | repository.generations[records[i].Name] = validGeneration(t, records[i]) | ||
| 469 | } | 469 | } | ||
| 470 | - | d, catalogue := newDurableRecoveryDaemon(t, records, repository) | 470 | + | d, _ := newDurableRecoveryDaemon(t, records, repository) |
| 471 | 471 | ||||
| 472 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 472 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 473 | 473 | ||||
| 474 | require.Len(t, d.inactive, count) | 474 | require.Len(t, d.inactive, count) | ||
| 475 | for _, record := range records { | 475 | for _, record := range records { | ||
| @@ -481,11 +481,11 @@ func TestCatalogueRestoreAggregateOver4096(t *testing.T) { | |||||
| 481 | func TestCatalogueRestoreSingleSessionOver4096(t *testing.T) { | 481 | func TestCatalogueRestoreSingleSessionOver4096(t *testing.T) { | ||
| 482 | record := durableRecoveryRecord(0) | 482 | record := durableRecoveryRecord(0) | ||
| 483 | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: validGeneration(t, record)}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | 483 | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: validGeneration(t, record)}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | ||
| 484 | - | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 484 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) |
| 485 | 485 | ||||
| 486 | // The repository may contain any number of stale entries; restoration has no | 486 | // The repository may contain any number of stale entries; restoration has no | ||
| 487 | // discovery API and therefore performs exactly one direct catalogue lookup. | 487 | // discovery API and therefore performs exactly one direct catalogue lookup. | ||
| 488 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 488 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 489 | 489 | ||||
| 490 | require.Equal(t, protocol.SessionDown, d.inactive[record.Name].state) | 490 | require.Equal(t, protocol.SessionDown, d.inactive[record.Name].state) | ||
| 491 | require.Equal(t, 1, repository.loads[record.Name]) | 491 | require.Equal(t, 1, repository.loads[record.Name]) | ||
| @@ -498,7 +498,7 @@ func TestCatalogueRestoreIncarnationMismatch(t *testing.T) { | |||||
| 498 | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: generation}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | 498 | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: generation}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | ||
| 499 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 499 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | ||
| 500 | 500 | ||||
| 501 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 501 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 502 | 502 | ||||
| 503 | require.Equal(t, protocol.SessionBroken, d.inactive[record.Name].state) | 503 | require.Equal(t, protocol.SessionBroken, d.inactive[record.Name].state) | ||
| 504 | got, _, _ := catalogue.Record(record.Name) | 504 | got, _, _ := catalogue.Record(record.Name) | ||
| @@ -674,7 +674,7 @@ func TestPersistAndRegisterRestoredSessionRegistrationFailurePreservesStoppedEnt | |||||
| 674 | sess := newSnapshotTestSession(t, record.Name, false, record.Cwd) | 674 | sess := newSnapshotTestSession(t, record.Name, false, record.Cwd) | ||
| 675 | sess.incarnation = record.IncarnationID | 675 | sess.incarnation = record.IncarnationID | ||
| 676 | 676 | ||||
| 677 | - | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess) | 677 | + | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess, d.purgeEpochSnapshot()) |
| 678 | 678 | ||||
| 679 | require.ErrorContains(t, err, "registry rejected") | 679 | require.ErrorContains(t, err, "registry rejected") | ||
| 680 | require.False(t, registered) | 680 | require.False(t, registered) | ||
| @@ -744,7 +744,7 @@ func TestPersistAndRegisterRestoredSessionRejectsReplacementIncarnation(t *testi | |||||
| 744 | sess := newSnapshotTestSession(t, record.Name, false, record.Cwd) | 744 | sess := newSnapshotTestSession(t, record.Name, false, record.Cwd) | ||
| 745 | sess.incarnation = record.IncarnationID | 745 | sess.incarnation = record.IncarnationID | ||
| 746 | 746 | ||||
| 747 | - | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess) | 747 | + | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess, d.purgeEpochSnapshot()) |
| 748 | require.ErrorContains(t, err, "incarnation") | 748 | require.ErrorContains(t, err, "incarnation") | ||
| 749 | require.False(t, registered) | 749 | require.False(t, registered) | ||
| 750 | require.Empty(t, d.sessions) | 750 | require.Empty(t, d.sessions) | ||
| @@ -842,7 +842,7 @@ func TestAttachBarrierSurvivesLiveRegistryPublication(t *testing.T) { | |||||
| 842 | d.mu.Unlock() | 842 | d.mu.Unlock() | ||
| 843 | sess := &session{sessionCore: sessionCore{name: record.Name, incarnation: record.IncarnationID}} | 843 | sess := &session{sessionCore: sessionCore{name: record.Name, incarnation: record.IncarnationID}} | ||
| 844 | 844 | ||||
| 845 | - | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess) | 845 | + | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess, d.purgeEpochSnapshot()) |
| 846 | require.NoError(t, err) | 846 | require.NoError(t, err) | ||
| 847 | require.True(t, registered) | 847 | require.True(t, registered) | ||
| 848 | 848 | ||||
| @@ -879,7 +879,7 @@ func TestRestoreCancellationTransitionsBeforeAttachCompletion(t *testing.T) { | |||||
| 879 | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | 879 | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | ||
| 880 | restored := make(chan struct{}) | 880 | restored := make(chan struct{}) | ||
| 881 | go func() { | 881 | go func() { | ||
| 882 | - | d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue)) | 882 | + | d.restoreIncrementalSnapshots(restoreCtx) |
| 883 | close(restored) | 883 | close(restored) | ||
| 884 | }() | 884 | }() | ||
| 885 | <-repository.started | 885 | <-repository.started | ||
| @@ -990,9 +990,9 @@ func TestCatalogueRestoreDegradedVisible(t *testing.T) { | |||||
| 990 | record := durableRecoveryRecord(0) | 990 | record := durableRecoveryRecord(0) | ||
| 991 | record.DegradedReason = "checkpoint validation failed" | 991 | record.DegradedReason = "checkpoint validation failed" | ||
| 992 | repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | 992 | repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | ||
| 993 | - | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 993 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) |
| 994 | 994 | ||||
| 995 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 995 | + | d.restoreIncrementalSnapshots(context.Background()) |
| 996 | 996 | ||||
| 997 | require.Equal(t, protocol.SessionBroken, d.inactive[record.Name].state) | 997 | require.Equal(t, protocol.SessionBroken, d.inactive[record.Name].state) | ||
| 998 | require.Zero(t, repository.loads[record.Name]) | 998 | require.Zero(t, repository.loads[record.Name]) | ||
M internal/usecase/daemon/move_lifecycle.go +29 -4
| @@ -33,11 +33,27 @@ func (d *Daemon) reserveMoveLifecycles(source, destination *session) (*moveLifec | |||
| 33 | 33 | } | |
| 34 | 34 | d.mu.Lock() | |
| 35 | 35 | closing := d.closing | |
| 36 | + | purgeClosing := d.purgeAdmissionClosing | |
| 37 | + | if !closing && !purgeClosing { | |
| 38 | + | // A move transfers live state across the exact lifecycle set a KillAll | |
| 39 | + | // purge captured, so it reserves the purge admission gate too: the purge | |
| 40 | + | // drains this reservation before it snapshots, and a move arriving during | |
| 41 | + | // the purge is rejected instead of racing the set. | |
| 42 | + | d.purgeAdmissionActive++ | |
| 43 | + | } | |
| 36 | 44 | d.mu.Unlock() | |
| 37 | 45 | if closing { | |
| 38 | 46 | d.moveLifecycleMu.Unlock() | |
| 39 | 47 | return nil, errMoveLifecycleUnavailable | |
| 40 | 48 | } | |
| 49 | + | if purgeClosing { | |
| 50 | + | // A transient KillAll purge owns the exact lifecycle set, so no move can | |
| 51 | + | // reserve it. Unlike a shutdown this is retry state, so report the same | |
| 52 | + | // typed admission outcome as the other command paths instead of a | |
| 53 | + | // permanent "target gone" verdict. | |
| 54 | + | d.moveLifecycleMu.Unlock() | |
| 55 | + | return nil, errPurgeAdmissionClosed | |
| 56 | + | } | |
| 41 | 57 | d.moveLifecycleActive++ | |
| 42 | 58 | d.signalMoveLifecycleChangedLocked() | |
| 43 | 59 | d.moveLifecycleMu.Unlock() | |
| @@ -53,8 +69,7 @@ func (d *Daemon) reserveMoveLifecycles(source, destination *session) (*moveLifec | |||
| 53 | 69 | if sess.teardownActive { | |
| 54 | 70 | unlockMoveSessions(sessions) | |
| 55 | 71 | d.moveLifecycleMu.Lock() | |
| 56 | - | d.moveLifecycleActive-- | |
| 57 | - | d.signalMoveLifecycleChangedLocked() | |
| 72 | + | d.releaseMoveAdmissionLocked() | |
| 58 | 73 | d.moveLifecycleMu.Unlock() | |
| 59 | 74 | return nil, errMoveLifecycleUnavailable | |
| 60 | 75 | } | |
| @@ -99,8 +114,7 @@ func (r *moveLifecycleReservation) Release() { | |||
| 99 | 114 | sess.signalTeardownChangedLocked() | |
| 100 | 115 | } | |
| 101 | 116 | unlockMoveSessions(r.sessions) | |
| 102 | - | r.daemon.moveLifecycleActive-- | |
| 103 | - | r.daemon.signalMoveLifecycleChangedLocked() | |
| 117 | + | r.daemon.releaseMoveAdmissionLocked() | |
| 104 | 118 | r.daemon.moveLifecycleMu.Unlock() | |
| 105 | 119 | }) | |
| 106 | 120 | } | |
| @@ -139,6 +153,17 @@ func (d *Daemon) closeMoveLifecycles() { | |||
| 139 | 153 | } | |
| 140 | 154 | } | |
| 141 | 155 | ||
| 156 | + | // releaseMoveAdmissionLocked drops one move's move-gate and KillAll purge-gate | |
| 157 | + | // reservations. Caller holds moveLifecycleMu; the purge gate is guarded by mu. | |
| 158 | + | func (d *Daemon) releaseMoveAdmissionLocked() { | |
| 159 | + | d.moveLifecycleActive-- | |
| 160 | + | d.signalMoveLifecycleChangedLocked() | |
| 161 | + | d.mu.Lock() | |
| 162 | + | d.purgeAdmissionActive-- | |
| 163 | + | d.signalPurgeAdmissionChangedLocked() | |
| 164 | + | d.mu.Unlock() | |
| 165 | + | } | |
| 166 | + | ||
| 142 | 167 | func (d *Daemon) moveLifecycleChangeLocked() chan struct{} { | |
| 143 | 168 | if d.moveLifecycleChanged == nil { | |
| 144 | 169 | d.moveLifecycleChanged = make(chan struct{}) | |
| @@ -33,11 +33,27 @@ func (d *Daemon) reserveMoveLifecycles(source, destination *session) (*moveLifec | |||||
| 33 | } | 33 | } | ||
| 34 | d.mu.Lock() | 34 | d.mu.Lock() | ||
| 35 | closing := d.closing | 35 | closing := d.closing | ||
| 36 | + | purgeClosing := d.purgeAdmissionClosing | |||
| 37 | + | if !closing && !purgeClosing { | |||
| 38 | + | // A move transfers live state across the exact lifecycle set a KillAll | |||
| 39 | + | // purge captured, so it reserves the purge admission gate too: the purge | |||
| 40 | + | // drains this reservation before it snapshots, and a move arriving during | |||
| 41 | + | // the purge is rejected instead of racing the set. | |||
| 42 | + | d.purgeAdmissionActive++ | |||
| 43 | + | } | |||
| 36 | d.mu.Unlock() | 44 | d.mu.Unlock() | ||
| 37 | if closing { | 45 | if closing { | ||
| 38 | d.moveLifecycleMu.Unlock() | 46 | d.moveLifecycleMu.Unlock() | ||
| 39 | return nil, errMoveLifecycleUnavailable | 47 | return nil, errMoveLifecycleUnavailable | ||
| 40 | } | 48 | } | ||
| 49 | + | if purgeClosing { | |||
| 50 | + | // A transient KillAll purge owns the exact lifecycle set, so no move can | |||
| 51 | + | // reserve it. Unlike a shutdown this is retry state, so report the same | |||
| 52 | + | // typed admission outcome as the other command paths instead of a | |||
| 53 | + | // permanent "target gone" verdict. | |||
| 54 | + | d.moveLifecycleMu.Unlock() | |||
| 55 | + | return nil, errPurgeAdmissionClosed | |||
| 56 | + | } | |||
| 41 | d.moveLifecycleActive++ | 57 | d.moveLifecycleActive++ | ||
| 42 | d.signalMoveLifecycleChangedLocked() | 58 | d.signalMoveLifecycleChangedLocked() | ||
| 43 | d.moveLifecycleMu.Unlock() | 59 | d.moveLifecycleMu.Unlock() | ||
| @@ -53,8 +69,7 @@ func (d *Daemon) reserveMoveLifecycles(source, destination *session) (*moveLifec | |||||
| 53 | if sess.teardownActive { | 69 | if sess.teardownActive { | ||
| 54 | unlockMoveSessions(sessions) | 70 | unlockMoveSessions(sessions) | ||
| 55 | d.moveLifecycleMu.Lock() | 71 | d.moveLifecycleMu.Lock() | ||
| 56 | - | d.moveLifecycleActive-- | 72 | + | d.releaseMoveAdmissionLocked() |
| 57 | - | d.signalMoveLifecycleChangedLocked() | |||
| 58 | d.moveLifecycleMu.Unlock() | 73 | d.moveLifecycleMu.Unlock() | ||
| 59 | return nil, errMoveLifecycleUnavailable | 74 | return nil, errMoveLifecycleUnavailable | ||
| 60 | } | 75 | } | ||
| @@ -99,8 +114,7 @@ func (r *moveLifecycleReservation) Release() { | |||||
| 99 | sess.signalTeardownChangedLocked() | 114 | sess.signalTeardownChangedLocked() | ||
| 100 | } | 115 | } | ||
| 101 | unlockMoveSessions(r.sessions) | 116 | unlockMoveSessions(r.sessions) | ||
| 102 | - | r.daemon.moveLifecycleActive-- | 117 | + | r.daemon.releaseMoveAdmissionLocked() |
| 103 | - | r.daemon.signalMoveLifecycleChangedLocked() | |||
| 104 | r.daemon.moveLifecycleMu.Unlock() | 118 | r.daemon.moveLifecycleMu.Unlock() | ||
| 105 | }) | 119 | }) | ||
| 106 | } | 120 | } | ||
| @@ -139,6 +153,17 @@ func (d *Daemon) closeMoveLifecycles() { | |||||
| 139 | } | 153 | } | ||
| 140 | } | 154 | } | ||
| 141 | 155 | ||||
| 156 | + | // releaseMoveAdmissionLocked drops one move's move-gate and KillAll purge-gate | |||
| 157 | + | // reservations. Caller holds moveLifecycleMu; the purge gate is guarded by mu. | |||
| 158 | + | func (d *Daemon) releaseMoveAdmissionLocked() { | |||
| 159 | + | d.moveLifecycleActive-- | |||
| 160 | + | d.signalMoveLifecycleChangedLocked() | |||
| 161 | + | d.mu.Lock() | |||
| 162 | + | d.purgeAdmissionActive-- | |||
| 163 | + | d.signalPurgeAdmissionChangedLocked() | |||
| 164 | + | d.mu.Unlock() | |||
| 165 | + | } | |||
| 166 | + | ||||
| 142 | func (d *Daemon) moveLifecycleChangeLocked() chan struct{} { | 167 | func (d *Daemon) moveLifecycleChangeLocked() chan struct{} { | ||
| 143 | if d.moveLifecycleChanged == nil { | 168 | if d.moveLifecycleChanged == nil { | ||
| 144 | d.moveLifecycleChanged = make(chan struct{}) | 169 | d.moveLifecycleChanged = make(chan struct{}) | ||
M internal/usecase/daemon/move_lifecycle_test.go +20 -0
| @@ -147,6 +147,26 @@ func TestMoveLifecycleReservationRejectsTeardownWithoutLeakingCounts(t *testing. | |||
| 147 | 147 | } | |
| 148 | 148 | } | |
| 149 | 149 | ||
| 150 | + | // TestMoveLifecycleReservationRejectedByPurgeAdmissionIsRetryable proves a | |
| 151 | + | // move that arrives while a KillAll purge owns the exact lifecycle set is | |
| 152 | + | // rejected with the transient purge-admission sentinel rather than the | |
| 153 | + | // permanent shutdown verdict, and that the rejection leaks no move or purge | |
| 154 | + | // admission counts. | |
| 155 | + | func TestMoveLifecycleReservationRejectedByPurgeAdmissionIsRetryable(t *testing.T) { | |
| 156 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 157 | + | source := &session{sessionCore: sessionCore{id: "source"}} | |
| 158 | + | destination := &session{sessionCore: sessionCore{id: "destination"}} | |
| 159 | + | ||
| 160 | + | beginTestPurgeAdmission(t, d) | |
| 161 | + | reservation, err := d.reserveMoveLifecycles(source, destination) | |
| 162 | + | require.ErrorIs(t, err, errPurgeAdmissionClosed) | |
| 163 | + | require.NotErrorIs(t, err, errMoveLifecycleUnavailable) | |
| 164 | + | require.Nil(t, reservation) | |
| 165 | + | awaitDaemonMoveActive(t, d, 0) | |
| 166 | + | d.endPurgeAdmission() | |
| 167 | + | requirePurgeAdmissionBalanced(t, d) | |
| 168 | + | } | |
| 169 | + | ||
| 150 | 170 | func TestMoveLifecycleReservationDeduplicatesAndReleasesExactlyOnce(t *testing.T) { | |
| 151 | 171 | d := newTestDaemon(t, nil, stubClock{}) | |
| 152 | 172 | sess := &session{sessionCore: sessionCore{id: "same"}} | |
| @@ -147,6 +147,26 @@ func TestMoveLifecycleReservationRejectsTeardownWithoutLeakingCounts(t *testing. | |||||
| 147 | } | 147 | } | ||
| 148 | } | 148 | } | ||
| 149 | 149 | ||||
| 150 | + | // TestMoveLifecycleReservationRejectedByPurgeAdmissionIsRetryable proves a | |||
| 151 | + | // move that arrives while a KillAll purge owns the exact lifecycle set is | |||
| 152 | + | // rejected with the transient purge-admission sentinel rather than the | |||
| 153 | + | // permanent shutdown verdict, and that the rejection leaks no move or purge | |||
| 154 | + | // admission counts. | |||
| 155 | + | func TestMoveLifecycleReservationRejectedByPurgeAdmissionIsRetryable(t *testing.T) { | |||
| 156 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 157 | + | source := &session{sessionCore: sessionCore{id: "source"}} | |||
| 158 | + | destination := &session{sessionCore: sessionCore{id: "destination"}} | |||
| 159 | + | ||||
| 160 | + | beginTestPurgeAdmission(t, d) | |||
| 161 | + | reservation, err := d.reserveMoveLifecycles(source, destination) | |||
| 162 | + | require.ErrorIs(t, err, errPurgeAdmissionClosed) | |||
| 163 | + | require.NotErrorIs(t, err, errMoveLifecycleUnavailable) | |||
| 164 | + | require.Nil(t, reservation) | |||
| 165 | + | awaitDaemonMoveActive(t, d, 0) | |||
| 166 | + | d.endPurgeAdmission() | |||
| 167 | + | requirePurgeAdmissionBalanced(t, d) | |||
| 168 | + | } | |||
| 169 | + | ||||
| 150 | func TestMoveLifecycleReservationDeduplicatesAndReleasesExactlyOnce(t *testing.T) { | 170 | func TestMoveLifecycleReservationDeduplicatesAndReleasesExactlyOnce(t *testing.T) { | ||
| 151 | d := newTestDaemon(t, nil, stubClock{}) | 171 | d := newTestDaemon(t, nil, stubClock{}) | ||
| 152 | sess := &session{sessionCore: sessionCore{id: "same"}} | 172 | sess := &session{sessionCore: sessionCore{id: "same"}} | ||
M internal/usecase/daemon/move_rejection.go +11 -0
| @@ -114,6 +114,11 @@ func movePickerUserError(err error) error { | |||
| 114 | 114 | if errors.As(err, &userErr) { | |
| 115 | 115 | return err | |
| 116 | 116 | } | |
| 117 | + | if errors.Is(err, errPurgeAdmissionClosed) { | |
| 118 | + | // A transient KillAll admission rejection is retry state: surface a | |
| 119 | + | // retryable warning instead of a permanent "Move failed" verdict. | |
| 120 | + | return domain.UserWarn(domain.NoticeSessionUnavailable, sessionKillRetryMessage, err) | |
| 121 | + | } | |
| 117 | 122 | descriptor, ok := moveRejectionDescriptorFor(err) | |
| 118 | 123 | if !ok { | |
| 119 | 124 | descriptor = moveRejectionDescriptorByReason(moveRejectionInvalid) | |
| @@ -145,6 +150,12 @@ func normalizeMoveRejection(err error) error { | |||
| 145 | 150 | if errors.As(err, &rejection) { | |
| 146 | 151 | return err | |
| 147 | 152 | } | |
| 153 | + | if errors.Is(err, errPurgeAdmissionClosed) { | |
| 154 | + | // A transient KillAll admission rejection is retry state, not a move | |
| 155 | + | // rejection: preserve the typed sentinel so the command and palette | |
| 156 | + | // adapters map it consistently with the other command paths. | |
| 157 | + | return err | |
| 158 | + | } | |
| 148 | 159 | if errors.Is(err, layout.ErrTooSmall) { | |
| 149 | 160 | return errMoveTooSmall | |
| 150 | 161 | } | |
| @@ -114,6 +114,11 @@ func movePickerUserError(err error) error { | |||||
| 114 | if errors.As(err, &userErr) { | 114 | if errors.As(err, &userErr) { | ||
| 115 | return err | 115 | return err | ||
| 116 | } | 116 | } | ||
| 117 | + | if errors.Is(err, errPurgeAdmissionClosed) { | |||
| 118 | + | // A transient KillAll admission rejection is retry state: surface a | |||
| 119 | + | // retryable warning instead of a permanent "Move failed" verdict. | |||
| 120 | + | return domain.UserWarn(domain.NoticeSessionUnavailable, sessionKillRetryMessage, err) | |||
| 121 | + | } | |||
| 117 | descriptor, ok := moveRejectionDescriptorFor(err) | 122 | descriptor, ok := moveRejectionDescriptorFor(err) | ||
| 118 | if !ok { | 123 | if !ok { | ||
| 119 | descriptor = moveRejectionDescriptorByReason(moveRejectionInvalid) | 124 | descriptor = moveRejectionDescriptorByReason(moveRejectionInvalid) | ||
| @@ -145,6 +150,12 @@ func normalizeMoveRejection(err error) error { | |||||
| 145 | if errors.As(err, &rejection) { | 150 | if errors.As(err, &rejection) { | ||
| 146 | return err | 151 | return err | ||
| 147 | } | 152 | } | ||
| 153 | + | if errors.Is(err, errPurgeAdmissionClosed) { | |||
| 154 | + | // A transient KillAll admission rejection is retry state, not a move | |||
| 155 | + | // rejection: preserve the typed sentinel so the command and palette | |||
| 156 | + | // adapters map it consistently with the other command paths. | |||
| 157 | + | return err | |||
| 158 | + | } | |||
| 148 | if errors.Is(err, layout.ErrTooSmall) { | 159 | if errors.Is(err, layout.ErrTooSmall) { | ||
| 149 | return errMoveTooSmall | 160 | return errMoveTooSmall | ||
| 150 | } | 161 | } | ||
M internal/usecase/daemon/move_rejection_test.go +20 -0
| @@ -50,6 +50,26 @@ func TestNormalizeMoveRejectionDoesNotLeakInternalErrors(t *testing.T) { | |||
| 50 | 50 | require.ErrorIs(t, got, errMovePaneInvalid) | |
| 51 | 51 | } | |
| 52 | 52 | ||
| 53 | + | // TestMoveRejectionPreservesTransientPurgeAdmission proves a transient KillAll | |
| 54 | + | // admission rejection is not wrapped as a move rejection or leaked as an | |
| 55 | + | // internal detail: the command adapter keeps the typed sentinel it maps | |
| 56 | + | // retryably, and the palette adapter renders a retryable warning. | |
| 57 | + | func TestMoveRejectionPreservesTransientPurgeAdmission(t *testing.T) { | |
| 58 | + | normalized := normalizeMoveRejection(errPurgeAdmissionClosed) | |
| 59 | + | require.ErrorIs(t, normalized, errPurgeAdmissionClosed) | |
| 60 | + | require.NotErrorIs(t, normalized, errMovePaneInvalid) | |
| 61 | + | var rejection *moveRejection | |
| 62 | + | require.False(t, errors.As(normalized, &rejection)) | |
| 63 | + | ||
| 64 | + | paletteErr := movePickerUserError(errPurgeAdmissionClosed) | |
| 65 | + | var userErr *domain.UserError | |
| 66 | + | require.ErrorAs(t, paletteErr, &userErr) | |
| 67 | + | require.Equal(t, domain.NoticeSessionUnavailable, userErr.Code) | |
| 68 | + | require.Equal(t, domain.NoticeWarn, userErr.Severity) | |
| 69 | + | require.Equal(t, sessionKillRetryMessage, userErr.Msg) | |
| 70 | + | require.ErrorIs(t, userErr.Err, errPurgeAdmissionClosed) | |
| 71 | + | } | |
| 72 | + | ||
| 53 | 73 | func TestMoveRejectionPresentationParity(t *testing.T) { | |
| 54 | 74 | tests := []struct { | |
| 55 | 75 | name string | |
| @@ -50,6 +50,26 @@ func TestNormalizeMoveRejectionDoesNotLeakInternalErrors(t *testing.T) { | |||||
| 50 | require.ErrorIs(t, got, errMovePaneInvalid) | 50 | require.ErrorIs(t, got, errMovePaneInvalid) | ||
| 51 | } | 51 | } | ||
| 52 | 52 | ||||
| 53 | + | // TestMoveRejectionPreservesTransientPurgeAdmission proves a transient KillAll | |||
| 54 | + | // admission rejection is not wrapped as a move rejection or leaked as an | |||
| 55 | + | // internal detail: the command adapter keeps the typed sentinel it maps | |||
| 56 | + | // retryably, and the palette adapter renders a retryable warning. | |||
| 57 | + | func TestMoveRejectionPreservesTransientPurgeAdmission(t *testing.T) { | |||
| 58 | + | normalized := normalizeMoveRejection(errPurgeAdmissionClosed) | |||
| 59 | + | require.ErrorIs(t, normalized, errPurgeAdmissionClosed) | |||
| 60 | + | require.NotErrorIs(t, normalized, errMovePaneInvalid) | |||
| 61 | + | var rejection *moveRejection | |||
| 62 | + | require.False(t, errors.As(normalized, &rejection)) | |||
| 63 | + | ||||
| 64 | + | paletteErr := movePickerUserError(errPurgeAdmissionClosed) | |||
| 65 | + | var userErr *domain.UserError | |||
| 66 | + | require.ErrorAs(t, paletteErr, &userErr) | |||
| 67 | + | require.Equal(t, domain.NoticeSessionUnavailable, userErr.Code) | |||
| 68 | + | require.Equal(t, domain.NoticeWarn, userErr.Severity) | |||
| 69 | + | require.Equal(t, sessionKillRetryMessage, userErr.Msg) | |||
| 70 | + | require.ErrorIs(t, userErr.Err, errPurgeAdmissionClosed) | |||
| 71 | + | } | |||
| 72 | + | ||||
| 53 | func TestMoveRejectionPresentationParity(t *testing.T) { | 73 | func TestMoveRejectionPresentationParity(t *testing.T) { | ||
| 54 | tests := []struct { | 74 | tests := []struct { | ||
| 55 | name string | 75 | name string | ||
M internal/usecase/daemon/move_transaction.go +7 -0
| @@ -2,6 +2,7 @@ package daemon | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | + | "errors" | |
| 5 | 6 | ||
| 6 | 7 | "github.com/bnema/vev/internal/domain" | |
| 7 | 8 | ) | |
| @@ -110,6 +111,12 @@ func (d *Daemon) executeMove(topology moveTopology) (result error) { | |||
| 110 | 111 | ||
| 111 | 112 | reservation, err := d.reserveMoveLifecycles(t.source, t.destination) | |
| 112 | 113 | if err != nil { | |
| 114 | + | // Preserve a typed transient admission rejection (a KillAll purge owning | |
| 115 | + | // the exact set) so the command path maps it retryably instead of | |
| 116 | + | // misreporting the destination as gone. | |
| 117 | + | if errors.Is(err, errPurgeAdmissionClosed) { | |
| 118 | + | return err | |
| 119 | + | } | |
| 113 | 120 | return errMoveStaleTarget | |
| 114 | 121 | } | |
| 115 | 122 | reservationHeld := true | |
| @@ -2,6 +2,7 @@ package daemon | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "errors" | |||
| 5 | 6 | ||||
| 6 | "github.com/bnema/vev/internal/domain" | 7 | "github.com/bnema/vev/internal/domain" | ||
| 7 | ) | 8 | ) | ||
| @@ -110,6 +111,12 @@ func (d *Daemon) executeMove(topology moveTopology) (result error) { | |||||
| 110 | 111 | ||||
| 111 | reservation, err := d.reserveMoveLifecycles(t.source, t.destination) | 112 | reservation, err := d.reserveMoveLifecycles(t.source, t.destination) | ||
| 112 | if err != nil { | 113 | if err != nil { | ||
| 114 | + | // Preserve a typed transient admission rejection (a KillAll purge owning | |||
| 115 | + | // the exact set) so the command path maps it retryably instead of | |||
| 116 | + | // misreporting the destination as gone. | |||
| 117 | + | if errors.Is(err, errPurgeAdmissionClosed) { | |||
| 118 | + | return err | |||
| 119 | + | } | |||
| 113 | return errMoveStaleTarget | 120 | return errMoveStaleTarget | ||
| 114 | } | 121 | } | ||
| 115 | reservationHeld := true | 122 | reservationHeld := true | ||
M internal/usecase/daemon/notify.go +21 -7
| @@ -182,12 +182,20 @@ func noticeDetails(err error) string { | |||
| 182 | 182 | return strings.Join(parts, " ← ") | |
| 183 | 183 | } | |
| 184 | 184 | ||
| 185 | - | // benignNoticeError reports errors that are expected control flow rather than | |
| 186 | - | // something the user needs to be told about. | |
| 185 | + | // benignNoticeError reports unclassified errors that are expected control flow | |
| 186 | + | // rather than something the user needs to be told about. A caller that wraps one | |
| 187 | + | // of these in an explicit *domain.UserError still reports its own message; this | |
| 188 | + | // filter only keeps an unclassified occurrence from surfacing as an | |
| 189 | + | // internal-error notice. | |
| 187 | 190 | func benignNoticeError(err error) bool { | |
| 188 | 191 | return errors.Is(err, context.Canceled) || | |
| 189 | 192 | errors.Is(err, errNoNeighbor) || | |
| 190 | - | errors.Is(err, ports.ErrNoClipboardImage) | |
| 193 | + | errors.Is(err, ports.ErrNoClipboardImage) || | |
| 194 | + | // A concurrent detach or token resume that invalidated a teardown's | |
| 195 | + | // participant snapshot is expected control flow: the caller observes an | |
| 196 | + | // explicit error and retries or reports it, so it must not also surface | |
| 197 | + | // as an internal-error notice. | |
| 198 | + | errors.Is(err, errSessionKillParticipantsChanged) | |
| 191 | 199 | } | |
| 192 | 200 | ||
| 193 | 201 | // persistShutdownSnapshotFailure records a terminal snapshot failure that | |
| @@ -222,17 +230,23 @@ func (d *Daemon) persistShutdownSnapshotFailure(name string, cause error) { | |||
| 222 | 230 | } | |
| 223 | 231 | } | |
| 224 | 232 | ||
| 225 | - | // reportError turns any error into a user-facing notice. Unclassified errors | |
| 226 | - | // become NoticeInternal: an error reaching here is never silently dropped | |
| 227 | - | // unless benignNoticeError says it is routine. | |
| 233 | + | // reportError turns any error into a user-facing notice. An explicitly | |
| 234 | + | // classified *domain.UserError always wins: the caller already decided what the | |
| 235 | + | // user should see, so it is reported even when the cause chain also matches a | |
| 236 | + | // routine sentinel (for example a retryable teardown abort). Any other error is | |
| 237 | + | // dropped only when benignNoticeError says it is routine; unclassified errors | |
| 238 | + | // become NoticeInternal. | |
| 228 | 239 | func (d *Daemon) reportError(sess *session, err error) { | |
| 229 | - | if err == nil || benignNoticeError(err) { | |
| 240 | + | if err == nil { | |
| 230 | 241 | return | |
| 231 | 242 | } | |
| 232 | 243 | if ue, ok := errors.AsType[*domain.UserError](err); ok { | |
| 233 | 244 | d.notify(sess, ue.Severity, ue.Code, ue.Msg, ue.Err) | |
| 234 | 245 | return | |
| 235 | 246 | } | |
| 247 | + | if benignNoticeError(err) { | |
| 248 | + | return | |
| 249 | + | } | |
| 236 | 250 | d.notify(sess, domain.NoticeError, domain.NoticeInternal, "internal error", err) | |
| 237 | 251 | } | |
| 238 | 252 | ||
| @@ -182,12 +182,20 @@ func noticeDetails(err error) string { | |||||
| 182 | return strings.Join(parts, " ← ") | 182 | return strings.Join(parts, " ← ") | ||
| 183 | } | 183 | } | ||
| 184 | 184 | ||||
| 185 | - | // benignNoticeError reports errors that are expected control flow rather than | 185 | + | // benignNoticeError reports unclassified errors that are expected control flow |
| 186 | - | // something the user needs to be told about. | 186 | + | // rather than something the user needs to be told about. A caller that wraps one |
| 187 | + | // of these in an explicit *domain.UserError still reports its own message; this | |||
| 188 | + | // filter only keeps an unclassified occurrence from surfacing as an | |||
| 189 | + | // internal-error notice. | |||
| 187 | func benignNoticeError(err error) bool { | 190 | func benignNoticeError(err error) bool { | ||
| 188 | return errors.Is(err, context.Canceled) || | 191 | return errors.Is(err, context.Canceled) || | ||
| 189 | errors.Is(err, errNoNeighbor) || | 192 | errors.Is(err, errNoNeighbor) || | ||
| 190 | - | errors.Is(err, ports.ErrNoClipboardImage) | 193 | + | errors.Is(err, ports.ErrNoClipboardImage) || |
| 194 | + | // A concurrent detach or token resume that invalidated a teardown's | |||
| 195 | + | // participant snapshot is expected control flow: the caller observes an | |||
| 196 | + | // explicit error and retries or reports it, so it must not also surface | |||
| 197 | + | // as an internal-error notice. | |||
| 198 | + | errors.Is(err, errSessionKillParticipantsChanged) | |||
| 191 | } | 199 | } | ||
| 192 | 200 | ||||
| 193 | // persistShutdownSnapshotFailure records a terminal snapshot failure that | 201 | // persistShutdownSnapshotFailure records a terminal snapshot failure that | ||
| @@ -222,17 +230,23 @@ func (d *Daemon) persistShutdownSnapshotFailure(name string, cause error) { | |||||
| 222 | } | 230 | } | ||
| 223 | } | 231 | } | ||
| 224 | 232 | ||||
| 225 | - | // reportError turns any error into a user-facing notice. Unclassified errors | 233 | + | // reportError turns any error into a user-facing notice. An explicitly |
| 226 | - | // become NoticeInternal: an error reaching here is never silently dropped | 234 | + | // classified *domain.UserError always wins: the caller already decided what the |
| 227 | - | // unless benignNoticeError says it is routine. | 235 | + | // user should see, so it is reported even when the cause chain also matches a |
| 236 | + | // routine sentinel (for example a retryable teardown abort). Any other error is | |||
| 237 | + | // dropped only when benignNoticeError says it is routine; unclassified errors | |||
| 238 | + | // become NoticeInternal. | |||
| 228 | func (d *Daemon) reportError(sess *session, err error) { | 239 | func (d *Daemon) reportError(sess *session, err error) { | ||
| 229 | - | if err == nil || benignNoticeError(err) { | 240 | + | if err == nil { |
| 230 | return | 241 | return | ||
| 231 | } | 242 | } | ||
| 232 | if ue, ok := errors.AsType[*domain.UserError](err); ok { | 243 | if ue, ok := errors.AsType[*domain.UserError](err); ok { | ||
| 233 | d.notify(sess, ue.Severity, ue.Code, ue.Msg, ue.Err) | 244 | d.notify(sess, ue.Severity, ue.Code, ue.Msg, ue.Err) | ||
| 234 | return | 245 | return | ||
| 235 | } | 246 | } | ||
| 247 | + | if benignNoticeError(err) { | |||
| 248 | + | return | |||
| 249 | + | } | |||
| 236 | d.notify(sess, domain.NoticeError, domain.NoticeInternal, "internal error", err) | 250 | d.notify(sess, domain.NoticeError, domain.NoticeInternal, "internal error", err) | ||
| 237 | } | 251 | } | ||
| 238 | 252 | ||||
M internal/usecase/daemon/notify_test.go +18 -0
| @@ -938,3 +938,21 @@ func TestToastExpiryRemovesOnlyTheEntryItArmed(t *testing.T) { | |||
| 938 | 938 | return len(ns) == 0 | |
| 939 | 939 | }, 200*time.Millisecond, 5*time.Millisecond, "expiry must remove only the entry it armed") | |
| 940 | 940 | } | |
| 941 | + | ||
| 942 | + | // TestReportErrorClassifiedUserErrorWinsOverBenignCause proves an explicitly | |
| 943 | + | // classified user error is reported even when its cause chain also matches a | |
| 944 | + | // routine sentinel. A retryable teardown abort must reach the user as its own | |
| 945 | + | // message instead of being filtered as internal control flow. | |
| 946 | + | func TestReportErrorClassifiedUserErrorWinsOverBenignCause(t *testing.T) { | |
| 947 | + | d, sess, ac, _ := newNoticeFixture(t, newNoticeClock()) | |
| 948 | + | ||
| 949 | + | d.reportError(sess, closeTabFailureNotice(fmt.Errorf("close tab: %w", errSessionKillParticipantsChanged))) | |
| 950 | + | ||
| 951 | + | hist := d.notices.history() | |
| 952 | + | require.Len(t, hist, 1) | |
| 953 | + | require.Contains(t, hist[0].Message, "couldn't close tab") | |
| 954 | + | require.Contains(t, hist[0].Message, "try again") | |
| 955 | + | ||
| 956 | + | toasts := awaitToastCount(t, ac, 1) | |
| 957 | + | require.Contains(t, toasts[0].Message, "try again") | |
| 958 | + | } | |
| @@ -938,3 +938,21 @@ func TestToastExpiryRemovesOnlyTheEntryItArmed(t *testing.T) { | |||||
| 938 | return len(ns) == 0 | 938 | return len(ns) == 0 | ||
| 939 | }, 200*time.Millisecond, 5*time.Millisecond, "expiry must remove only the entry it armed") | 939 | }, 200*time.Millisecond, 5*time.Millisecond, "expiry must remove only the entry it armed") | ||
| 940 | } | 940 | } | ||
| 941 | + | ||||
| 942 | + | // TestReportErrorClassifiedUserErrorWinsOverBenignCause proves an explicitly | |||
| 943 | + | // classified user error is reported even when its cause chain also matches a | |||
| 944 | + | // routine sentinel. A retryable teardown abort must reach the user as its own | |||
| 945 | + | // message instead of being filtered as internal control flow. | |||
| 946 | + | func TestReportErrorClassifiedUserErrorWinsOverBenignCause(t *testing.T) { | |||
| 947 | + | d, sess, ac, _ := newNoticeFixture(t, newNoticeClock()) | |||
| 948 | + | ||||
| 949 | + | d.reportError(sess, closeTabFailureNotice(fmt.Errorf("close tab: %w", errSessionKillParticipantsChanged))) | |||
| 950 | + | ||||
| 951 | + | hist := d.notices.history() | |||
| 952 | + | require.Len(t, hist, 1) | |||
| 953 | + | require.Contains(t, hist[0].Message, "couldn't close tab") | |||
| 954 | + | require.Contains(t, hist[0].Message, "try again") | |||
| 955 | + | ||||
| 956 | + | toasts := awaitToastCount(t, ac, 1) | |||
| 957 | + | require.Contains(t, toasts[0].Message, "try again") | |||
| 958 | + | } | |||
M internal/usecase/daemon/pane_actions.go +16 -5
| @@ -315,17 +315,28 @@ func (d *Daemon) reapTiledPaneLease(lease paneEffectLease) bool { | |||
| 315 | 315 | ac.overlays.clearCopyModeForPane(p) | |
| 316 | 316 | } | |
| 317 | 317 | } | |
| 318 | + | // A concurrent detach or token resume that invalidated the teardown's | |
| 319 | + | // participant snapshot is itself observable progress: the next owner | |
| 320 | + | // resolution consumes the fresh participant set, so retry instead of | |
| 321 | + | // treating the abort as a completed close. | |
| 322 | + | participantsChanged := false | |
| 318 | 323 | if err := d.closeTabLocked(sess, tb, true); err != nil { | |
| 319 | - | return true | |
| 324 | + | if !errors.Is(err, errSessionKillParticipantsChanged) { | |
| 325 | + | return true | |
| 326 | + | } | |
| 327 | + | participantsChanged = true | |
| 320 | 328 | } | |
| 321 | 329 | if p.owner.Load() == nil { | |
| 322 | 330 | return true | |
| 323 | 331 | } | |
| 324 | 332 | // A successful no-op teardown means participant, gate, or lifecycle | |
| 325 | 333 | // validation rejected the mutation without changing either the tab list | |
| 326 | - | // or the pane owner. Retrying that exact state would spin forever; only | |
| 327 | - | // retry when the teardown made observable progress that can be consumed | |
| 328 | - | // by the next owner resolution. | |
| 334 | + | // or the pane owner. That includes an explicit participants-changed | |
| 335 | + | // sentinel: the kill aborted because a concurrent detach or transport | |
| 336 | + | // replacement invalidated its snapshot, which the owner resolution below | |
| 337 | + | // consumes. Retrying that exact state would spin forever; only retry when | |
| 338 | + | // the teardown made observable progress that can be consumed by the next | |
| 339 | + | // owner resolution. | |
| 329 | 340 | d.mu.Lock() | |
| 330 | 341 | sessionCurrent := d.sessions[sess.id] == sess | |
| 331 | 342 | sess.mu.Lock() | |
| @@ -341,7 +352,7 @@ func (d *Daemon) reapTiledPaneLease(lease paneEffectLease) bool { | |||
| 341 | 352 | } | |
| 342 | 353 | sess.mu.Unlock() | |
| 343 | 354 | d.mu.Unlock() | |
| 344 | - | if sessionCurrent && tabCurrent && !attachmentsChanged { | |
| 355 | + | if sessionCurrent && tabCurrent && !attachmentsChanged && !participantsChanged { | |
| 345 | 356 | return true | |
| 346 | 357 | } | |
| 347 | 358 | return false | |
| @@ -315,17 +315,28 @@ func (d *Daemon) reapTiledPaneLease(lease paneEffectLease) bool { | |||||
| 315 | ac.overlays.clearCopyModeForPane(p) | 315 | ac.overlays.clearCopyModeForPane(p) | ||
| 316 | } | 316 | } | ||
| 317 | } | 317 | } | ||
| 318 | + | // A concurrent detach or token resume that invalidated the teardown's | |||
| 319 | + | // participant snapshot is itself observable progress: the next owner | |||
| 320 | + | // resolution consumes the fresh participant set, so retry instead of | |||
| 321 | + | // treating the abort as a completed close. | |||
| 322 | + | participantsChanged := false | |||
| 318 | if err := d.closeTabLocked(sess, tb, true); err != nil { | 323 | if err := d.closeTabLocked(sess, tb, true); err != nil { | ||
| 319 | - | return true | 324 | + | if !errors.Is(err, errSessionKillParticipantsChanged) { |
| 325 | + | return true | |||
| 326 | + | } | |||
| 327 | + | participantsChanged = true | |||
| 320 | } | 328 | } | ||
| 321 | if p.owner.Load() == nil { | 329 | if p.owner.Load() == nil { | ||
| 322 | return true | 330 | return true | ||
| 323 | } | 331 | } | ||
| 324 | // A successful no-op teardown means participant, gate, or lifecycle | 332 | // A successful no-op teardown means participant, gate, or lifecycle | ||
| 325 | // validation rejected the mutation without changing either the tab list | 333 | // validation rejected the mutation without changing either the tab list | ||
| 326 | - | // or the pane owner. Retrying that exact state would spin forever; only | 334 | + | // or the pane owner. That includes an explicit participants-changed |
| 327 | - | // retry when the teardown made observable progress that can be consumed | 335 | + | // sentinel: the kill aborted because a concurrent detach or transport |
| 328 | - | // by the next owner resolution. | 336 | + | // replacement invalidated its snapshot, which the owner resolution below |
| 337 | + | // consumes. Retrying that exact state would spin forever; only retry when | |||
| 338 | + | // the teardown made observable progress that can be consumed by the next | |||
| 339 | + | // owner resolution. | |||
| 329 | d.mu.Lock() | 340 | d.mu.Lock() | ||
| 330 | sessionCurrent := d.sessions[sess.id] == sess | 341 | sessionCurrent := d.sessions[sess.id] == sess | ||
| 331 | sess.mu.Lock() | 342 | sess.mu.Lock() | ||
| @@ -341,7 +352,7 @@ func (d *Daemon) reapTiledPaneLease(lease paneEffectLease) bool { | |||||
| 341 | } | 352 | } | ||
| 342 | sess.mu.Unlock() | 353 | sess.mu.Unlock() | ||
| 343 | d.mu.Unlock() | 354 | d.mu.Unlock() | ||
| 344 | - | if sessionCurrent && tabCurrent && !attachmentsChanged { | 355 | + | if sessionCurrent && tabCurrent && !attachmentsChanged && !participantsChanged { |
| 345 | return true | 356 | return true | ||
| 346 | } | 357 | } | ||
| 347 | return false | 358 | return false | ||
M internal/usecase/daemon/picker.go +6 -0
| @@ -726,6 +726,9 @@ func (d *Daemon) resumeStoppedAndSwitchLocked(from *session, ac *attachedClient, | |||
| 726 | 726 | expectedTransport: sourceCapability.transport, sourceCapability: &sourceCapability, sourceEffect: sourceEffect, action: action, | |
| 727 | 727 | expectedSourceTab: guard.expectedSource, copySourceEnvironment: true, ready: true, | |
| 728 | 728 | createTargetLocked: func() (*session, error) { | |
| 729 | + | if d.purgeAdmissionClosedLocked() { | |
| 730 | + | return nil, errPurgeAdmissionClosed | |
| 731 | + | } | |
| 729 | 732 | current, ok := d.inactive[target.Name] | |
| 730 | 733 | if !ok || !current.canResume() || !current.sameLifecycle(stopped) || !targetMatchesLifecycle(target, current.name, current.createdAt, current.incarnation) { | |
| 731 | 734 | return nil, errAttachmentTransition | |
| @@ -759,6 +762,9 @@ func (d *Daemon) resumeStoppedAndSwitchLocked(from *session, ac *attachedClient, | |||
| 759 | 762 | } | |
| 760 | 763 | env := copyEnvironment(from.env) | |
| 761 | 764 | from.mu.Unlock() | |
| 765 | + | if d.purgeAdmissionClosedLocked() { | |
| 766 | + | return nil, attachmentTransitionResult{}, false, errPurgeAdmissionClosed | |
| 767 | + | } | |
| 762 | 768 | current, ok := d.inactive[target.Name] | |
| 763 | 769 | if !ok || !current.canResume() || !current.sameLifecycle(stopped) || !targetMatchesLifecycle(target, current.name, current.createdAt, current.incarnation) { | |
| 764 | 770 | return nil, attachmentTransitionResult{}, false, errAttachmentTransition | |
| @@ -726,6 +726,9 @@ func (d *Daemon) resumeStoppedAndSwitchLocked(from *session, ac *attachedClient, | |||||
| 726 | expectedTransport: sourceCapability.transport, sourceCapability: &sourceCapability, sourceEffect: sourceEffect, action: action, | 726 | expectedTransport: sourceCapability.transport, sourceCapability: &sourceCapability, sourceEffect: sourceEffect, action: action, | ||
| 727 | expectedSourceTab: guard.expectedSource, copySourceEnvironment: true, ready: true, | 727 | expectedSourceTab: guard.expectedSource, copySourceEnvironment: true, ready: true, | ||
| 728 | createTargetLocked: func() (*session, error) { | 728 | createTargetLocked: func() (*session, error) { | ||
| 729 | + | if d.purgeAdmissionClosedLocked() { | |||
| 730 | + | return nil, errPurgeAdmissionClosed | |||
| 731 | + | } | |||
| 729 | current, ok := d.inactive[target.Name] | 732 | current, ok := d.inactive[target.Name] | ||
| 730 | if !ok || !current.canResume() || !current.sameLifecycle(stopped) || !targetMatchesLifecycle(target, current.name, current.createdAt, current.incarnation) { | 733 | if !ok || !current.canResume() || !current.sameLifecycle(stopped) || !targetMatchesLifecycle(target, current.name, current.createdAt, current.incarnation) { | ||
| 731 | return nil, errAttachmentTransition | 734 | return nil, errAttachmentTransition | ||
| @@ -759,6 +762,9 @@ func (d *Daemon) resumeStoppedAndSwitchLocked(from *session, ac *attachedClient, | |||||
| 759 | } | 762 | } | ||
| 760 | env := copyEnvironment(from.env) | 763 | env := copyEnvironment(from.env) | ||
| 761 | from.mu.Unlock() | 764 | from.mu.Unlock() | ||
| 765 | + | if d.purgeAdmissionClosedLocked() { | |||
| 766 | + | return nil, attachmentTransitionResult{}, false, errPurgeAdmissionClosed | |||
| 767 | + | } | |||
| 762 | current, ok := d.inactive[target.Name] | 768 | current, ok := d.inactive[target.Name] | ||
| 763 | if !ok || !current.canResume() || !current.sameLifecycle(stopped) || !targetMatchesLifecycle(target, current.name, current.createdAt, current.incarnation) { | 769 | if !ok || !current.canResume() || !current.sameLifecycle(stopped) || !targetMatchesLifecycle(target, current.name, current.createdAt, current.incarnation) { | ||
| 764 | return nil, attachmentTransitionResult{}, false, errAttachmentTransition | 770 | return nil, attachmentTransitionResult{}, false, errAttachmentTransition | ||
A internal/usecase/daemon/purge.go +681 -0
| @@ -0,0 +1,681 @@ | |||
| 1 | + | package daemon | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "errors" | |
| 6 | + | "fmt" | |
| 7 | + | "sort" | |
| 8 | + | "strings" | |
| 9 | + | "sync" | |
| 10 | + | ||
| 11 | + | "github.com/bnema/vev/internal/domain" | |
| 12 | + | ) | |
| 13 | + | ||
| 14 | + | // errPurgeAdmissionClosed is returned to command-path creation/resume that | |
| 15 | + | // already holds architecture locks and therefore cannot wait for a daemon-wide | |
| 16 | + | // purge. Route-level creation waits on the purge admission gate instead. | |
| 17 | + | var errPurgeAdmissionClosed = errors.New("daemon is purging sessions") | |
| 18 | + | ||
| 19 | + | // purgeAdmissionClosedLocked reports whether a KillAll purge currently owns | |
| 20 | + | // admission. Caller holds mu. A command-path create/resume that cannot wait | |
| 21 | + | // rejects with errPurgeAdmissionClosed instead of inserting into the exact set | |
| 22 | + | // being removed. | |
| 23 | + | func (d *Daemon) purgeAdmissionClosedLocked() bool { return d.purgeAdmissionClosing } | |
| 24 | + | ||
| 25 | + | // acquirePurgeAdmission reserves one create/restore transition against the | |
| 26 | + | // transient KillAll gate. It waits without holding any other lock while a purge | |
| 27 | + | // owns the gate, so a Hello arriving mid-purge is admitted as a create or | |
| 28 | + | // restore after the purge rather than inserted into the set being removed. | |
| 29 | + | func (d *Daemon) acquirePurgeAdmission(ctx context.Context) error { | |
| 30 | + | for { | |
| 31 | + | d.mu.Lock() | |
| 32 | + | if !d.purgeAdmissionClosing { | |
| 33 | + | d.purgeAdmissionActive++ | |
| 34 | + | d.mu.Unlock() | |
| 35 | + | return nil | |
| 36 | + | } | |
| 37 | + | changed := d.purgeAdmissionChangeLocked() | |
| 38 | + | d.mu.Unlock() | |
| 39 | + | select { | |
| 40 | + | case <-changed: | |
| 41 | + | case <-ctx.Done(): | |
| 42 | + | return ctx.Err() | |
| 43 | + | } | |
| 44 | + | } | |
| 45 | + | } | |
| 46 | + | ||
| 47 | + | // releasePurgeAdmission drops a reservation taken by acquirePurgeAdmission or | |
| 48 | + | // an equivalent move reservation. | |
| 49 | + | func (d *Daemon) releasePurgeAdmission() { | |
| 50 | + | d.mu.Lock() | |
| 51 | + | d.purgeAdmissionActive-- | |
| 52 | + | d.signalPurgeAdmissionChangedLocked() | |
| 53 | + | d.mu.Unlock() | |
| 54 | + | } | |
| 55 | + | ||
| 56 | + | // purgeAdmissionStatus reports how a KillAll's attempt to close the transient | |
| 57 | + | // admission gate resolved. | |
| 58 | + | type purgeAdmissionStatus uint8 | |
| 59 | + | ||
| 60 | + | const ( | |
| 61 | + | // purgeAdmissionGranted means the gate closed and every admitted transition | |
| 62 | + | // drained within the bound, so the purge owns the exact lifecycle set. | |
| 63 | + | purgeAdmissionGranted purgeAdmissionStatus = iota | |
| 64 | + | // purgeAdmissionSuperseded means an explicit daemon shutdown began, or the | |
| 65 | + | // caller's context ended, before the drain completed. The purge removes | |
| 66 | + | // nothing and defers to the stop. | |
| 67 | + | purgeAdmissionSuperseded | |
| 68 | + | // purgeAdmissionTimedOut means an admitted transition did not release the | |
| 69 | + | // gate before the bounded purge budget expired. The purge removes nothing so | |
| 70 | + | // a transition stuck in repository I/O that ignores cancellation can never | |
| 71 | + | // wedge Serve. | |
| 72 | + | purgeAdmissionTimedOut | |
| 73 | + | ) | |
| 74 | + | ||
| 75 | + | // beginPurgeAdmission closes the KillAll admission gate and waits, bounded, for | |
| 76 | + | // every admitted transition to drain before the caller snapshots the exact | |
| 77 | + | // lifecycle set. It mirrors closeMoveLifecycles but is re-openable: the caller | |
| 78 | + | // must always call endPurgeAdmission to clear it and leave the daemon serving. | |
| 79 | + | // | |
| 80 | + | // The wait ends as soon as the drain completes. It also ends early, without the | |
| 81 | + | // purge owning anything, when ctx is done, an explicit shutdown begins | |
| 82 | + | // (closing), or deadline expires — so an admitted create, restore, or move that | |
| 83 | + | // ignores cancellation (for example a repository call that never returns) can | |
| 84 | + | // never extend Serve past the bounded purge budget. On success purgeEpoch is | |
| 85 | + | // incremented so a restoration worker that waited on the gate can detect that | |
| 86 | + | // the record set it captured was superseded. | |
| 87 | + | func (d *Daemon) beginPurgeAdmission(ctx context.Context, deadline *snapshotShutdownDeadline) purgeAdmissionStatus { | |
| 88 | + | if ctx == nil { | |
| 89 | + | ctx = context.Background() | |
| 90 | + | } | |
| 91 | + | d.mu.Lock() | |
| 92 | + | d.purgeAdmissionClosing = true | |
| 93 | + | d.signalPurgeAdmissionChangedLocked() | |
| 94 | + | for { | |
| 95 | + | if d.closing { | |
| 96 | + | d.mu.Unlock() | |
| 97 | + | return purgeAdmissionSuperseded | |
| 98 | + | } | |
| 99 | + | if d.purgeAdmissionActive == 0 { | |
| 100 | + | d.purgeEpoch++ | |
| 101 | + | d.signalPurgeAdmissionChangedLocked() | |
| 102 | + | d.mu.Unlock() | |
| 103 | + | return purgeAdmissionGranted | |
| 104 | + | } | |
| 105 | + | changed := d.purgeAdmissionChangeLocked() | |
| 106 | + | d.mu.Unlock() | |
| 107 | + | select { | |
| 108 | + | case <-changed: | |
| 109 | + | case <-ctx.Done(): | |
| 110 | + | return purgeAdmissionSuperseded | |
| 111 | + | case <-deadline.Done(): | |
| 112 | + | // An explicit stop that began as the budget expired wins: classify it | |
| 113 | + | // as superseded so the purge defers and reports stop semantics instead | |
| 114 | + | // of a drain timeout. terminateAllForShutdown also signals changed, but | |
| 115 | + | // a ready select may still pick the deadline, so recheck closing here. | |
| 116 | + | d.mu.Lock() | |
| 117 | + | closing := d.closing | |
| 118 | + | d.mu.Unlock() | |
| 119 | + | if closing { | |
| 120 | + | return purgeAdmissionSuperseded | |
| 121 | + | } | |
| 122 | + | return purgeAdmissionTimedOut | |
| 123 | + | } | |
| 124 | + | d.mu.Lock() | |
| 125 | + | } | |
| 126 | + | } | |
| 127 | + | ||
| 128 | + | func (d *Daemon) endPurgeAdmission() { | |
| 129 | + | d.mu.Lock() | |
| 130 | + | d.purgeAdmissionClosing = false | |
| 131 | + | d.signalPurgeAdmissionChangedLocked() | |
| 132 | + | d.mu.Unlock() | |
| 133 | + | } | |
| 134 | + | ||
| 135 | + | // purgeEpochSnapshot returns the current purge boundary. Restoration captures | |
| 136 | + | // it before reading catalogue records so a purge that completes while its | |
| 137 | + | // workers wait for admission invalidates those records. | |
| 138 | + | func (d *Daemon) purgeEpochSnapshot() uint64 { | |
| 139 | + | d.mu.Lock() | |
| 140 | + | defer d.mu.Unlock() | |
| 141 | + | return d.purgeEpoch | |
| 142 | + | } | |
| 143 | + | ||
| 144 | + | // purgeSupersededLocked reports whether an admitted transition captured at | |
| 145 | + | // epoch must abandon its work: an explicit shutdown has begun, a purge currently | |
| 146 | + | // owns and waits on admission, or a purge completed after epoch was captured. | |
| 147 | + | // Caller holds mu. | |
| 148 | + | func (d *Daemon) purgeSupersededLocked(epoch uint64) bool { | |
| 149 | + | return d.closing || d.purgeAdmissionClosing || d.purgeEpoch != epoch | |
| 150 | + | } | |
| 151 | + | ||
| 152 | + | // purgeSuperseded is purgeSupersededLocked without the caller-held lock. | |
| 153 | + | func (d *Daemon) purgeSuperseded(epoch uint64) bool { | |
| 154 | + | d.mu.Lock() | |
| 155 | + | defer d.mu.Unlock() | |
| 156 | + | return d.purgeSupersededLocked(epoch) | |
| 157 | + | } | |
| 158 | + | ||
| 159 | + | // purgeSupersededForShutdown reports whether an explicit daemon shutdown has | |
| 160 | + | // begun. A KillAll purge defers to it so explicit stop semantics (preserving | |
| 161 | + | // live sessions as stopped authority) are never overwritten by a concurrent | |
| 162 | + | // purge. | |
| 163 | + | func (d *Daemon) purgeSupersededForShutdown() bool { | |
| 164 | + | d.mu.Lock() | |
| 165 | + | defer d.mu.Unlock() | |
| 166 | + | return d.closing | |
| 167 | + | } | |
| 168 | + | ||
| 169 | + | // purgeAdmissionChangeLocked returns the channel closed when either purge | |
| 170 | + | // admission field changes. Caller holds mu. | |
| 171 | + | func (d *Daemon) purgeAdmissionChangeLocked() chan struct{} { | |
| 172 | + | if d.purgeAdmissionChanged == nil { | |
| 173 | + | d.purgeAdmissionChanged = make(chan struct{}) | |
| 174 | + | } | |
| 175 | + | return d.purgeAdmissionChanged | |
| 176 | + | } | |
| 177 | + | ||
| 178 | + | // signalPurgeAdmissionChangedLocked wakes every admission waiter. Caller holds | |
| 179 | + | // mu. | |
| 180 | + | func (d *Daemon) signalPurgeAdmissionChangedLocked() { | |
| 181 | + | close(d.purgeAdmissionChangeLocked()) | |
| 182 | + | d.purgeAdmissionChanged = make(chan struct{}) | |
| 183 | + | } | |
| 184 | + | ||
| 185 | + | // purgeRecordClass identifies which lifecycle record class a failed purge unit | |
| 186 | + | // belongs to, so callers can report a typed partial failure instead of one | |
| 187 | + | // opaque error. | |
| 188 | + | type purgeRecordClass uint8 | |
| 189 | + | ||
| 190 | + | const ( | |
| 191 | + | purgeRecordLive purgeRecordClass = iota + 1 | |
| 192 | + | purgeRecordStopped | |
| 193 | + | purgeRecordBroken | |
| 194 | + | purgeRecordDurable | |
| 195 | + | ) | |
| 196 | + | ||
| 197 | + | func (c purgeRecordClass) String() string { | |
| 198 | + | switch c { | |
| 199 | + | case purgeRecordLive: | |
| 200 | + | return "live" | |
| 201 | + | case purgeRecordStopped: | |
| 202 | + | return "stopped" | |
| 203 | + | case purgeRecordBroken: | |
| 204 | + | return "broken" | |
| 205 | + | case purgeRecordDurable: | |
| 206 | + | return "durable" | |
| 207 | + | default: | |
| 208 | + | return "unknown" | |
| 209 | + | } | |
| 210 | + | } | |
| 211 | + | ||
| 212 | + | // inactivePurgeClass resolves the record class of a captured stopped/broken | |
| 213 | + | // registry entry. | |
| 214 | + | func inactivePurgeClass(entry inactiveSession) purgeRecordClass { | |
| 215 | + | if entry.broken() { | |
| 216 | + | return purgeRecordBroken | |
| 217 | + | } | |
| 218 | + | return purgeRecordStopped | |
| 219 | + | } | |
| 220 | + | ||
| 221 | + | // purgeFailure is one structured partial purge failure: the record class, the | |
| 222 | + | // affected name when known, and the underlying error. | |
| 223 | + | type purgeFailure struct { | |
| 224 | + | Class purgeRecordClass | |
| 225 | + | Name string | |
| 226 | + | Err error | |
| 227 | + | } | |
| 228 | + | ||
| 229 | + | func (f purgeFailure) Error() string { | |
| 230 | + | if f.Name == "" { | |
| 231 | + | return fmt.Sprintf("%s: %v", f.Class, f.Err) | |
| 232 | + | } | |
| 233 | + | return fmt.Sprintf("%s %q: %v", f.Class, f.Name, f.Err) | |
| 234 | + | } | |
| 235 | + | ||
| 236 | + | func (f purgeFailure) Unwrap() error { return f.Err } | |
| 237 | + | ||
| 238 | + | // purgeResult reports every record that could not be fully purged. An empty | |
| 239 | + | // Failures slice means the purge removed the entire exact lifecycle set. A | |
| 240 | + | // superseded result means an explicit daemon shutdown won the race: the purge | |
| 241 | + | // deferred and removed nothing, or removed only the units whose teardown it had | |
| 242 | + | // already owned when the stop began. A drain timeout result means an admitted | |
| 243 | + | // transition refused to drain within the bounded admission budget, so the purge | |
| 244 | + | // removed nothing and reopened admission. A typed live failure means a live unit | |
| 245 | + | // was left untouched because the shared live-teardown budget expired before its | |
| 246 | + | // destructive boundary or because its participants kept changing. A typed | |
| 247 | + | // stopped/broken failure means the bounded stopped/broken sweep did not finish | |
| 248 | + | // the record's durable purge within its own budget. | |
| 249 | + | type purgeResult struct { | |
| 250 | + | Failures []purgeFailure | |
| 251 | + | superseded bool | |
| 252 | + | drainTimedOut bool | |
| 253 | + | } | |
| 254 | + | ||
| 255 | + | func (r purgeResult) Failed() bool { return len(r.Failures) > 0 || r.drainTimedOut } | |
| 256 | + | ||
| 257 | + | // Superseded reports that an explicit daemon shutdown owned the lifecycle set | |
| 258 | + | // before this purge could finish destroying it, so the purge deferred to it. | |
| 259 | + | // Any Failures it carries describe units the stop still owns; a superseded | |
| 260 | + | // result never means the purge silently reported success. | |
| 261 | + | func (r purgeResult) Superseded() bool { return r.superseded } | |
| 262 | + | ||
| 263 | + | // purgeSummaryFailureLimit bounds the wire-safe failure detail. The result may | |
| 264 | + | // hold arbitrarily many failures (one per record), but a control response must | |
| 265 | + | // stay bounded: report the total count and only the first N failures, then note | |
| 266 | + | // how many were omitted. | |
| 267 | + | const purgeSummaryFailureLimit = 5 | |
| 268 | + | ||
| 269 | + | func (r purgeResult) summary() string { | |
| 270 | + | if r.drainTimedOut { | |
| 271 | + | return "session purge aborted: purge admission did not drain within the bounded timeout; no records were removed" | |
| 272 | + | } | |
| 273 | + | if !r.Failed() { | |
| 274 | + | return "" | |
| 275 | + | } | |
| 276 | + | shown := min(len(r.Failures), purgeSummaryFailureLimit) | |
| 277 | + | parts := make([]string, 0, shown) | |
| 278 | + | for _, failure := range r.Failures[:shown] { | |
| 279 | + | parts = append(parts, failure.Error()) | |
| 280 | + | } | |
| 281 | + | if omitted := len(r.Failures) - shown; omitted > 0 { | |
| 282 | + | return fmt.Sprintf("%d record(s) could not be purged; first %d: %s; %d more omitted", len(r.Failures), shown, strings.Join(parts, "; "), omitted) | |
| 283 | + | } | |
| 284 | + | return fmt.Sprintf("%d record(s) could not be purged: %s", len(r.Failures), strings.Join(parts, "; ")) | |
| 285 | + | } | |
| 286 | + | ||
| 287 | + | // purgeAllSessions is the KillAll operation: it removes every live session and | |
| 288 | + | // every stopped/broken/durable record while leaving the daemon active. It does | |
| 289 | + | // not set closing, close move admission, cancel daemon-wide pane/process | |
| 290 | + | // contexts, close done, or consume shutdown signals, and it never makes Serve | |
| 291 | + | // return. | |
| 292 | + | // | |
| 293 | + | // Linearization: beginPurgeAdmission closes the transient admission gate and | |
| 294 | + | // drains in-flight create/restore/move reservations, so the exact lifecycle set | |
| 295 | + | // captured below cannot be mutated by a concurrent transition. Purge then runs | |
| 296 | + | // against that exact set, and endPurgeAdmission reopens admission for later | |
| 297 | + | // creations, which are ordered after the purge. | |
| 298 | + | // | |
| 299 | + | // Budgets: the admission drain, the stopped/broken durable sweep, and | |
| 300 | + | // live-session teardown spend separate bounded budgets, each matching the | |
| 301 | + | // daemon shutdown budget. A transition that ignores | |
| 302 | + | // cancellation and burns the whole drain budget therefore cannot leave teardown | |
| 303 | + | // with no time, and teardown cannot extend the drain. A repository deletion | |
| 304 | + | // that ignores cancellation cannot extend the stopped/broken sweep past its own | |
| 305 | + | // budget either. The live budget is armed | |
| 306 | + | // when the live phase begins and is shared by every live unit: once it expires, | |
| 307 | + | // each remaining live unit is skipped and reported as a typed live failure | |
| 308 | + | // rather than reported as purged. Every unit keeps its own accurate name. | |
| 309 | + | func (d *Daemon) purgeAllSessions(reason uint8) purgeResult { | |
| 310 | + | d.purgeAllMu.Lock() | |
| 311 | + | defer d.purgeAllMu.Unlock() | |
| 312 | + | ||
| 313 | + | // One deadline bounds the drain of every admitted transition. It is stopped | |
| 314 | + | // before the destructive phase so teardown spends a fresh budget. | |
| 315 | + | admissionDeadline := newSnapshotShutdownDeadline(d.clock) | |
| 316 | + | defer admissionDeadline.stop() | |
| 317 | + | ||
| 318 | + | ctx := d.serveCtx | |
| 319 | + | if ctx == nil { | |
| 320 | + | ctx = context.Background() | |
| 321 | + | } | |
| 322 | + | switch d.beginPurgeAdmission(ctx, admissionDeadline) { | |
| 323 | + | case purgeAdmissionSuperseded: | |
| 324 | + | d.log.Info("session purge superseded before admission", "reason", reason) | |
| 325 | + | d.endPurgeAdmission() | |
| 326 | + | return purgeResult{superseded: true} | |
| 327 | + | case purgeAdmissionTimedOut: | |
| 328 | + | d.log.Warn("session purge aborted: purge admission did not drain", "reason", reason) | |
| 329 | + | d.endPurgeAdmission() | |
| 330 | + | return purgeResult{drainTimedOut: true} | |
| 331 | + | } | |
| 332 | + | admissionDeadline.stop() | |
| 333 | + | defer d.endPurgeAdmission() | |
| 334 | + | ||
| 335 | + | d.mu.Lock() | |
| 336 | + | // A shutdown that began after admission was granted still wins: no record | |
| 337 | + | // has been removed yet, so defer without touching the preserved set. | |
| 338 | + | if d.closing { | |
| 339 | + | d.mu.Unlock() | |
| 340 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "pre-snapshot") | |
| 341 | + | return purgeResult{superseded: true} | |
| 342 | + | } | |
| 343 | + | d.purgeAllParkingLocked() | |
| 344 | + | parkedRetirements := d.purgeAllParkedLocked() | |
| 345 | + | d.purgeAllSuspendedLocked() | |
| 346 | + | live := d.sessionsSnapshotLocked() | |
| 347 | + | inactive := make([]inactiveSession, 0, len(d.inactive)) | |
| 348 | + | exact := make(map[string]struct{}, len(live)+len(d.inactive)) | |
| 349 | + | for _, entry := range d.inactive { | |
| 350 | + | inactive = append(inactive, entry) | |
| 351 | + | exact[entry.name] = struct{}{} | |
| 352 | + | } | |
| 353 | + | for _, sess := range live { | |
| 354 | + | sess.mu.Lock() | |
| 355 | + | exact[sess.name] = struct{}{} | |
| 356 | + | sess.mu.Unlock() | |
| 357 | + | } | |
| 358 | + | d.mu.Unlock() | |
| 359 | + | if d.afterPurgeAdmission != nil { | |
| 360 | + | d.afterPurgeAdmission() | |
| 361 | + | } | |
| 362 | + | d.finishParkedAttachmentRetirements(parkedRetirements) | |
| 363 | + | sort.Slice(inactive, func(i, j int) bool { return inactive[i].name < inactive[j].name }) | |
| 364 | + | ||
| 365 | + | // A shutdown that began while the snapshot was taken still wins: no record | |
| 366 | + | // has been removed yet, so defer without touching the preserved set. | |
| 367 | + | if d.purgeSupersededForShutdown() { | |
| 368 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "pre-destructive") | |
| 369 | + | return purgeResult{superseded: true} | |
| 370 | + | } | |
| 371 | + | ||
| 372 | + | result := purgeResult{} | |
| 373 | + | d.log.Info("session purge begin", "reason", reason, "live_sessions", len(live), "inactive_sessions", len(inactive)) | |
| 374 | + | ||
| 375 | + | // Stopped authority first (mirroring the shutdown purge order), then live | |
| 376 | + | // sessions. Each unit is independent: one failure never cancels a sibling. | |
| 377 | + | // Explicit stop wins per unit: a shutdown that begins mid-purge stops further | |
| 378 | + | // removals so it can preserve every not-yet-purged lifecycle as stopped | |
| 379 | + | // authority. A unit the purge already owns (its teardown is in progress) | |
| 380 | + | // keeps purge authority and completes as a purge; only units not yet started | |
| 381 | + | // are handed back to the stop. | |
| 382 | + | // | |
| 383 | + | // The stopped/broken sweep spends its own deadline and context, separate from | |
| 384 | + | // the admission drain and the live-teardown budget. A repository deletion | |
| 385 | + | // that ignores cancellation therefore cannot extend the purge: at the budget | |
| 386 | + | // every record the sweep did not finish is still stopped/broken authority and | |
| 387 | + | // is reported as a typed failure, and admission is released promptly. | |
| 388 | + | if len(inactive) > 0 { | |
| 389 | + | stoppedDeadline := newSnapshotShutdownDeadline(d.clock) | |
| 390 | + | defer stoppedDeadline.stop() | |
| 391 | + | stoppedCtx, cancelStopped := snapshotStopContext(stoppedDeadline) | |
| 392 | + | defer cancelStopped() | |
| 393 | + | sweep := d.purgeInactiveSweep(inactive, stoppedCtx, stoppedDeadline) | |
| 394 | + | result.Failures = append(result.Failures, sweep.failures...) | |
| 395 | + | if sweep.superseded { | |
| 396 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "stopped") | |
| 397 | + | return purgeResult{Failures: result.Failures, superseded: true} | |
| 398 | + | } | |
| 399 | + | } | |
| 400 | + | // The live phase gets its own bounded budget, armed here so neither the | |
| 401 | + | // admission drain nor the stopped-record purge consumes it. It is shared by | |
| 402 | + | // every live unit. | |
| 403 | + | destructiveDeadline := newSnapshotShutdownDeadline(d.clock) | |
| 404 | + | defer destructiveDeadline.stop() | |
| 405 | + | for _, sess := range live { | |
| 406 | + | if d.purgeSupersededForShutdown() { | |
| 407 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "live") | |
| 408 | + | return purgeResult{Failures: result.Failures, superseded: true} | |
| 409 | + | } | |
| 410 | + | sess.mu.Lock() | |
| 411 | + | name := sess.name | |
| 412 | + | sess.mu.Unlock() | |
| 413 | + | if d.beforePurgeLiveSessionKill != nil { | |
| 414 | + | d.beforePurgeLiveSessionKill(sess) | |
| 415 | + | } | |
| 416 | + | sess.stopInMemoryLifecycle() | |
| 417 | + | err := d.purgeLiveSession(sess, reason, destructiveDeadline) | |
| 418 | + | if err == nil { | |
| 419 | + | // A stop that wins this unit after the precheck above makes | |
| 420 | + | // purgeLiveSession a no-op: the stop already removed the session and | |
| 421 | + | // now owns it. Recheck so the purge never counts a stop-owned unit as a | |
| 422 | + | // successful purge. A unit whose teardown the purge already owned keeps | |
| 423 | + | // purge authority and is reported as removed, but the remaining units | |
| 424 | + | // still belong to the stop. | |
| 425 | + | if d.purgeSupersededForShutdown() { | |
| 426 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "live-owned") | |
| 427 | + | return purgeResult{Failures: result.Failures, superseded: true} | |
| 428 | + | } | |
| 429 | + | continue | |
| 430 | + | } | |
| 431 | + | if d.purgeSupersededForShutdown() && | |
| 432 | + | (errors.Is(err, errSessionKillDeadline) || errors.Is(err, errSessionKillParticipantsChanged)) { | |
| 433 | + | // The explicit stop owns the remaining lifecycles: either the live budget | |
| 434 | + | // expired as it began, or a participant change aborted a kill that the | |
| 435 | + | // stop now classifies as preservation rather than a purge failure. | |
| 436 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "live-deadline") | |
| 437 | + | return purgeResult{Failures: result.Failures, superseded: true} | |
| 438 | + | } | |
| 439 | + | result.Failures = append(result.Failures, purgeFailure{Class: purgeRecordLive, Name: name, Err: err}) | |
| 440 | + | d.log.Error("purging live session failed", "err", err, "session", name) | |
| 441 | + | } | |
| 442 | + | result.Failures = append(result.Failures, d.purgeDurableResidue(exact, result.Failures)...) | |
| 443 | + | sort.SliceStable(result.Failures, func(i, j int) bool { | |
| 444 | + | if result.Failures[i].Class != result.Failures[j].Class { | |
| 445 | + | return result.Failures[i].Class < result.Failures[j].Class | |
| 446 | + | } | |
| 447 | + | return result.Failures[i].Name < result.Failures[j].Name | |
| 448 | + | }) | |
| 449 | + | if result.Failed() { | |
| 450 | + | d.log.Warn("session purge incomplete", "reason", reason, "failures", len(result.Failures)) | |
| 451 | + | } | |
| 452 | + | return result | |
| 453 | + | } | |
| 454 | + | ||
| 455 | + | // purgeDurableResidue removes durable records that have no live/stopped/broken | |
| 456 | + | // registry entry, so a KillAll still accounts for every catalogue record. A | |
| 457 | + | // record already represented by a reported live/stopped/broken failure is that | |
| 458 | + | // failure's durable residue and is not double-reported here. | |
| 459 | + | func (d *Daemon) purgeDurableResidue(exact map[string]struct{}, existing []purgeFailure) []purgeFailure { | |
| 460 | + | if !d.persistEnabled || d.catalogue == nil { | |
| 461 | + | return nil | |
| 462 | + | } | |
| 463 | + | records, err := d.catalogue.Records() | |
| 464 | + | if err != nil { | |
| 465 | + | return []purgeFailure{{Class: purgeRecordDurable, Err: err}} | |
| 466 | + | } | |
| 467 | + | reported := make(map[string]struct{}, len(existing)) | |
| 468 | + | for _, failure := range existing { | |
| 469 | + | reported[failure.Name] = struct{}{} | |
| 470 | + | } | |
| 471 | + | var out []purgeFailure | |
| 472 | + | for _, record := range records { | |
| 473 | + | if _, accounted := exact[record.Name]; accounted { | |
| 474 | + | continue | |
| 475 | + | } | |
| 476 | + | if _, already := reported[record.Name]; already { | |
| 477 | + | continue | |
| 478 | + | } | |
| 479 | + | if err := d.purgeDurableRecord(record); err != nil { | |
| 480 | + | out = append(out, purgeFailure{Class: purgeRecordDurable, Name: record.Name, Err: err}) | |
| 481 | + | } | |
| 482 | + | } | |
| 483 | + | return out | |
| 484 | + | } | |
| 485 | + | ||
| 486 | + | // errSessionPurgeSweepDeadline reports that the stopped/broken sweep's own | |
| 487 | + | // bounded budget expired before a record's durable purge finished. The record | |
| 488 | + | // keeps its stopped/broken authority and is reported as a typed failure rather | |
| 489 | + | // than silently counted as purged. | |
| 490 | + | var errSessionPurgeSweepDeadline = errors.New("stopped session purge sweep deadline exceeded") | |
| 491 | + | ||
| 492 | + | // errSessionPurgeDeletePending reports that a purge-owned live unit's exact | |
| 493 | + | // durable delete did not finish within the shared live-teardown budget. The | |
| 494 | + | // live identity is already removed and its stopped authority still reserves the | |
| 495 | + | // name; a detached worker finishes the fenced DeleteExact and removes that | |
| 496 | + | // authority once the repository returns. It is a typed failure so a bounded | |
| 497 | + | // purge never reports a skipped durable delete as success, and a repository | |
| 498 | + | // delete that ignores cancellation can never hold purgeAllMu, the admission | |
| 499 | + | // gate, or control responses. | |
| 500 | + | var errSessionPurgeDeletePending = errors.New("session durable purge pending") | |
| 501 | + | ||
| 502 | + | // finishLiveDurablePurge runs the exact durable delete for a live unit the purge | |
| 503 | + | // already removed from the live registry. With a bounded deadline it runs the | |
| 504 | + | // delete on a detached worker: the caller returns errSessionPurgeDeletePending | |
| 505 | + | // at the deadline while the worker finishes and removes the matching stopped | |
| 506 | + | // authority. That removal is fenced by the captured name, incarnation, and | |
| 507 | + | // createdAt, so a same-name or re-incarnated replacement is never touched. A | |
| 508 | + | // nil deadline (a command-path kill) keeps the previous synchronous behavior. | |
| 509 | + | func (d *Daemon) finishLiveDurablePurge(deadline *snapshotShutdownDeadline, record domain.CatalogueRecord) error { | |
| 510 | + | expected := inactiveSession{name: record.Name, createdAt: record.CreatedAt, incarnation: record.IncarnationID, purging: true} | |
| 511 | + | finish := func() error { | |
| 512 | + | if err := d.finishSnapshotPurge(d.serveCtx, record.Name, record.IncarnationID, record.CreatedAt); err != nil { | |
| 513 | + | return err | |
| 514 | + | } | |
| 515 | + | d.mu.Lock() | |
| 516 | + | if stopped, ok := d.inactive[record.Name]; ok && stopped.purging && stopped.sameLifecycle(expected) { | |
| 517 | + | delete(d.inactive, record.Name) | |
| 518 | + | } | |
| 519 | + | d.mu.Unlock() | |
| 520 | + | d.reconcileAllRouteHistories() | |
| 521 | + | return nil | |
| 522 | + | } | |
| 523 | + | if deadline == nil { | |
| 524 | + | return finish() | |
| 525 | + | } | |
| 526 | + | done := make(chan error, 1) | |
| 527 | + | go func() { done <- finish() }() | |
| 528 | + | select { | |
| 529 | + | case err := <-done: | |
| 530 | + | return err | |
| 531 | + | case <-deadline.Done(): | |
| 532 | + | // Recheck completion so a delete that finished as the budget expired is | |
| 533 | + | // reported as success rather than pending. | |
| 534 | + | select { | |
| 535 | + | case err := <-done: | |
| 536 | + | return err | |
| 537 | + | default: | |
| 538 | + | // The caller already reported a typed pending failure and stopped | |
| 539 | + | // waiting. Wait for the detached delete on its own goroutine so a late | |
| 540 | + | // failure is still recorded with bounded, non-sensitive fields (record | |
| 541 | + | // class, incarnation, created-time) instead of the user-chosen session | |
| 542 | + | // name. | |
| 543 | + | go func() { | |
| 544 | + | if err := <-done; err != nil { | |
| 545 | + | d.log.Warn("detached session purge delete failed after deadline", | |
| 546 | + | "class", purgeRecordLive, "incarnation", record.IncarnationID, "created_at", record.CreatedAt, "err", err) | |
| 547 | + | } | |
| 548 | + | }() | |
| 549 | + | return errSessionPurgeDeletePending | |
| 550 | + | } | |
| 551 | + | } | |
| 552 | + | } | |
| 553 | + | ||
| 554 | + | // purgeInactiveSweepResult reports the bounded stopped/broken sweep outcome. | |
| 555 | + | // failures holds every record the purge could not finish; superseded means an | |
| 556 | + | // explicit stop began mid-sweep and owns the remaining records. | |
| 557 | + | type purgeInactiveSweepResult struct { | |
| 558 | + | failures []purgeFailure | |
| 559 | + | superseded bool | |
| 560 | + | } | |
| 561 | + | ||
| 562 | + | // purgeInactiveSweep removes the captured stopped/broken records under its own | |
| 563 | + | // deadline and context, separate from the admission drain and the | |
| 564 | + | // live-teardown budget. Because a repository deletion may ignore cancellation, | |
| 565 | + | // the sweep runs on a detached worker and the caller returns at the deadline: | |
| 566 | + | // every record the worker did not finish keeps its stopped/broken authority and | |
| 567 | + | // is reported as a typed failure, so a bounded purge releases admission | |
| 568 | + | // promptly instead of waiting out an uncooperative delete. The detached worker | |
| 569 | + | // stops contributing to the returned result at that point; its registry removal | |
| 570 | + | // stays lifecycle-fenced, so a late completion cannot delete a same-name | |
| 571 | + | // replacement. | |
| 572 | + | func (d *Daemon) purgeInactiveSweep(inactive []inactiveSession, ctx context.Context, deadline *snapshotShutdownDeadline) purgeInactiveSweepResult { | |
| 573 | + | var ( | |
| 574 | + | mu sync.Mutex | |
| 575 | + | failures []purgeFailure | |
| 576 | + | completed int | |
| 577 | + | superseded bool | |
| 578 | + | detached bool | |
| 579 | + | ) | |
| 580 | + | sweepDone := make(chan struct{}) | |
| 581 | + | go func() { | |
| 582 | + | defer close(sweepDone) | |
| 583 | + | for _, entry := range inactive { | |
| 584 | + | mu.Lock() | |
| 585 | + | abandoned := detached | |
| 586 | + | mu.Unlock() | |
| 587 | + | if abandoned { | |
| 588 | + | return | |
| 589 | + | } | |
| 590 | + | if d.purgeSupersededForShutdown() { | |
| 591 | + | mu.Lock() | |
| 592 | + | superseded = true | |
| 593 | + | mu.Unlock() | |
| 594 | + | return | |
| 595 | + | } | |
| 596 | + | class := inactivePurgeClass(entry) | |
| 597 | + | err := d.retryStoppedPurgeContextExact(ctx, entry.name, entry.incarnation, &entry.createdAt) | |
| 598 | + | mu.Lock() | |
| 599 | + | if detached { | |
| 600 | + | // The bounded caller already reported this unit; a late result must | |
| 601 | + | // not mutate the returned result. | |
| 602 | + | mu.Unlock() | |
| 603 | + | return | |
| 604 | + | } | |
| 605 | + | if err != nil { | |
| 606 | + | failures = append(failures, purgeFailure{Class: class, Name: entry.name, Err: err}) | |
| 607 | + | d.log.Error("purging stopped session failed", "err", err, "session", entry.name) | |
| 608 | + | } | |
| 609 | + | completed++ | |
| 610 | + | mu.Unlock() | |
| 611 | + | } | |
| 612 | + | }() | |
| 613 | + | ||
| 614 | + | select { | |
| 615 | + | case <-sweepDone: | |
| 616 | + | case <-deadline.Done(): | |
| 617 | + | mu.Lock() | |
| 618 | + | detached = true | |
| 619 | + | done := completed | |
| 620 | + | reported := append([]purgeFailure(nil), failures...) | |
| 621 | + | mu.Unlock() | |
| 622 | + | if d.purgeSupersededForShutdown() { | |
| 623 | + | // An explicit stop that began as the budget expired owns the remaining | |
| 624 | + | // records, so the sweep is supersession rather than a purge failure. | |
| 625 | + | return purgeInactiveSweepResult{failures: reported, superseded: true} | |
| 626 | + | } | |
| 627 | + | for _, entry := range inactive[done:] { | |
| 628 | + | reported = append(reported, purgeFailure{Class: inactivePurgeClass(entry), Name: entry.name, Err: errSessionPurgeSweepDeadline}) | |
| 629 | + | d.log.Error("purging stopped session failed", "err", errSessionPurgeSweepDeadline, "session", entry.name) | |
| 630 | + | } | |
| 631 | + | return purgeInactiveSweepResult{failures: reported} | |
| 632 | + | } | |
| 633 | + | ||
| 634 | + | mu.Lock() | |
| 635 | + | defer mu.Unlock() | |
| 636 | + | return purgeInactiveSweepResult{failures: failures, superseded: superseded} | |
| 637 | + | } | |
| 638 | + | ||
| 639 | + | // purgeLiveSession removes one live session under the shared live-teardown | |
| 640 | + | // budget. A post-freeze participant change aborts the kill before it touches the | |
| 641 | + | // session. A session a racing path already removed owns nothing and counts as | |
| 642 | + | // purged; an explicit stop that began meanwhile owns the rest and is never | |
| 643 | + | // retried against; otherwise the purge retries once against a fresh snapshot, | |
| 644 | + | // and a session whose participants changed again stays a typed live failure for | |
| 645 | + | // the caller. | |
| 646 | + | func (d *Daemon) purgeLiveSession(sess *session, reason uint8, deadline *snapshotShutdownDeadline) error { | |
| 647 | + | err := d.killSessionWithSnapshotDeadline(sess, reason, true, deadline, nil) | |
| 648 | + | if !errors.Is(err, errSessionKillParticipantsChanged) { | |
| 649 | + | return err | |
| 650 | + | } | |
| 651 | + | if d.purgeSupersededForShutdown() { | |
| 652 | + | // An explicit stop has begun: it owns the session's remaining lifecycle, so | |
| 653 | + | // the purge must not retry against it. The caller classifies the abort as | |
| 654 | + | // supersession instead of a live failure. | |
| 655 | + | return err | |
| 656 | + | } | |
| 657 | + | if !d.sessionRegistered(sess) { | |
| 658 | + | return nil | |
| 659 | + | } | |
| 660 | + | return d.killSessionWithSnapshotDeadline(sess, reason, true, deadline, nil) | |
| 661 | + | } | |
| 662 | + | ||
| 663 | + | // sessionRegistered reports whether sess is still the exact live registry entry. | |
| 664 | + | func (d *Daemon) sessionRegistered(sess *session) bool { | |
| 665 | + | if sess == nil { | |
| 666 | + | return false | |
| 667 | + | } | |
| 668 | + | d.mu.Lock() | |
| 669 | + | defer d.mu.Unlock() | |
| 670 | + | return d.sessions[sess.id] == sess | |
| 671 | + | } | |
| 672 | + | ||
| 673 | + | func (d *Daemon) purgeDurableRecord(record domain.CatalogueRecord) error { | |
| 674 | + | if !d.persistEnabled { | |
| 675 | + | return nil | |
| 676 | + | } | |
| 677 | + | if d.recovery == nil { | |
| 678 | + | return errors.New("durable session authority is not configured") | |
| 679 | + | } | |
| 680 | + | return d.finishSnapshotPurge(d.serveCtx, record.Name, record.IncarnationID, record.CreatedAt) | |
| 681 | + | } | |
| @@ -0,0 +1,681 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "errors" | |||
| 6 | + | "fmt" | |||
| 7 | + | "sort" | |||
| 8 | + | "strings" | |||
| 9 | + | "sync" | |||
| 10 | + | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 12 | + | ) | |||
| 13 | + | ||||
| 14 | + | // errPurgeAdmissionClosed is returned to command-path creation/resume that | |||
| 15 | + | // already holds architecture locks and therefore cannot wait for a daemon-wide | |||
| 16 | + | // purge. Route-level creation waits on the purge admission gate instead. | |||
| 17 | + | var errPurgeAdmissionClosed = errors.New("daemon is purging sessions") | |||
| 18 | + | ||||
| 19 | + | // purgeAdmissionClosedLocked reports whether a KillAll purge currently owns | |||
| 20 | + | // admission. Caller holds mu. A command-path create/resume that cannot wait | |||
| 21 | + | // rejects with errPurgeAdmissionClosed instead of inserting into the exact set | |||
| 22 | + | // being removed. | |||
| 23 | + | func (d *Daemon) purgeAdmissionClosedLocked() bool { return d.purgeAdmissionClosing } | |||
| 24 | + | ||||
| 25 | + | // acquirePurgeAdmission reserves one create/restore transition against the | |||
| 26 | + | // transient KillAll gate. It waits without holding any other lock while a purge | |||
| 27 | + | // owns the gate, so a Hello arriving mid-purge is admitted as a create or | |||
| 28 | + | // restore after the purge rather than inserted into the set being removed. | |||
| 29 | + | func (d *Daemon) acquirePurgeAdmission(ctx context.Context) error { | |||
| 30 | + | for { | |||
| 31 | + | d.mu.Lock() | |||
| 32 | + | if !d.purgeAdmissionClosing { | |||
| 33 | + | d.purgeAdmissionActive++ | |||
| 34 | + | d.mu.Unlock() | |||
| 35 | + | return nil | |||
| 36 | + | } | |||
| 37 | + | changed := d.purgeAdmissionChangeLocked() | |||
| 38 | + | d.mu.Unlock() | |||
| 39 | + | select { | |||
| 40 | + | case <-changed: | |||
| 41 | + | case <-ctx.Done(): | |||
| 42 | + | return ctx.Err() | |||
| 43 | + | } | |||
| 44 | + | } | |||
| 45 | + | } | |||
| 46 | + | ||||
| 47 | + | // releasePurgeAdmission drops a reservation taken by acquirePurgeAdmission or | |||
| 48 | + | // an equivalent move reservation. | |||
| 49 | + | func (d *Daemon) releasePurgeAdmission() { | |||
| 50 | + | d.mu.Lock() | |||
| 51 | + | d.purgeAdmissionActive-- | |||
| 52 | + | d.signalPurgeAdmissionChangedLocked() | |||
| 53 | + | d.mu.Unlock() | |||
| 54 | + | } | |||
| 55 | + | ||||
| 56 | + | // purgeAdmissionStatus reports how a KillAll's attempt to close the transient | |||
| 57 | + | // admission gate resolved. | |||
| 58 | + | type purgeAdmissionStatus uint8 | |||
| 59 | + | ||||
| 60 | + | const ( | |||
| 61 | + | // purgeAdmissionGranted means the gate closed and every admitted transition | |||
| 62 | + | // drained within the bound, so the purge owns the exact lifecycle set. | |||
| 63 | + | purgeAdmissionGranted purgeAdmissionStatus = iota | |||
| 64 | + | // purgeAdmissionSuperseded means an explicit daemon shutdown began, or the | |||
| 65 | + | // caller's context ended, before the drain completed. The purge removes | |||
| 66 | + | // nothing and defers to the stop. | |||
| 67 | + | purgeAdmissionSuperseded | |||
| 68 | + | // purgeAdmissionTimedOut means an admitted transition did not release the | |||
| 69 | + | // gate before the bounded purge budget expired. The purge removes nothing so | |||
| 70 | + | // a transition stuck in repository I/O that ignores cancellation can never | |||
| 71 | + | // wedge Serve. | |||
| 72 | + | purgeAdmissionTimedOut | |||
| 73 | + | ) | |||
| 74 | + | ||||
| 75 | + | // beginPurgeAdmission closes the KillAll admission gate and waits, bounded, for | |||
| 76 | + | // every admitted transition to drain before the caller snapshots the exact | |||
| 77 | + | // lifecycle set. It mirrors closeMoveLifecycles but is re-openable: the caller | |||
| 78 | + | // must always call endPurgeAdmission to clear it and leave the daemon serving. | |||
| 79 | + | // | |||
| 80 | + | // The wait ends as soon as the drain completes. It also ends early, without the | |||
| 81 | + | // purge owning anything, when ctx is done, an explicit shutdown begins | |||
| 82 | + | // (closing), or deadline expires — so an admitted create, restore, or move that | |||
| 83 | + | // ignores cancellation (for example a repository call that never returns) can | |||
| 84 | + | // never extend Serve past the bounded purge budget. On success purgeEpoch is | |||
| 85 | + | // incremented so a restoration worker that waited on the gate can detect that | |||
| 86 | + | // the record set it captured was superseded. | |||
| 87 | + | func (d *Daemon) beginPurgeAdmission(ctx context.Context, deadline *snapshotShutdownDeadline) purgeAdmissionStatus { | |||
| 88 | + | if ctx == nil { | |||
| 89 | + | ctx = context.Background() | |||
| 90 | + | } | |||
| 91 | + | d.mu.Lock() | |||
| 92 | + | d.purgeAdmissionClosing = true | |||
| 93 | + | d.signalPurgeAdmissionChangedLocked() | |||
| 94 | + | for { | |||
| 95 | + | if d.closing { | |||
| 96 | + | d.mu.Unlock() | |||
| 97 | + | return purgeAdmissionSuperseded | |||
| 98 | + | } | |||
| 99 | + | if d.purgeAdmissionActive == 0 { | |||
| 100 | + | d.purgeEpoch++ | |||
| 101 | + | d.signalPurgeAdmissionChangedLocked() | |||
| 102 | + | d.mu.Unlock() | |||
| 103 | + | return purgeAdmissionGranted | |||
| 104 | + | } | |||
| 105 | + | changed := d.purgeAdmissionChangeLocked() | |||
| 106 | + | d.mu.Unlock() | |||
| 107 | + | select { | |||
| 108 | + | case <-changed: | |||
| 109 | + | case <-ctx.Done(): | |||
| 110 | + | return purgeAdmissionSuperseded | |||
| 111 | + | case <-deadline.Done(): | |||
| 112 | + | // An explicit stop that began as the budget expired wins: classify it | |||
| 113 | + | // as superseded so the purge defers and reports stop semantics instead | |||
| 114 | + | // of a drain timeout. terminateAllForShutdown also signals changed, but | |||
| 115 | + | // a ready select may still pick the deadline, so recheck closing here. | |||
| 116 | + | d.mu.Lock() | |||
| 117 | + | closing := d.closing | |||
| 118 | + | d.mu.Unlock() | |||
| 119 | + | if closing { | |||
| 120 | + | return purgeAdmissionSuperseded | |||
| 121 | + | } | |||
| 122 | + | return purgeAdmissionTimedOut | |||
| 123 | + | } | |||
| 124 | + | d.mu.Lock() | |||
| 125 | + | } | |||
| 126 | + | } | |||
| 127 | + | ||||
| 128 | + | func (d *Daemon) endPurgeAdmission() { | |||
| 129 | + | d.mu.Lock() | |||
| 130 | + | d.purgeAdmissionClosing = false | |||
| 131 | + | d.signalPurgeAdmissionChangedLocked() | |||
| 132 | + | d.mu.Unlock() | |||
| 133 | + | } | |||
| 134 | + | ||||
| 135 | + | // purgeEpochSnapshot returns the current purge boundary. Restoration captures | |||
| 136 | + | // it before reading catalogue records so a purge that completes while its | |||
| 137 | + | // workers wait for admission invalidates those records. | |||
| 138 | + | func (d *Daemon) purgeEpochSnapshot() uint64 { | |||
| 139 | + | d.mu.Lock() | |||
| 140 | + | defer d.mu.Unlock() | |||
| 141 | + | return d.purgeEpoch | |||
| 142 | + | } | |||
| 143 | + | ||||
| 144 | + | // purgeSupersededLocked reports whether an admitted transition captured at | |||
| 145 | + | // epoch must abandon its work: an explicit shutdown has begun, a purge currently | |||
| 146 | + | // owns and waits on admission, or a purge completed after epoch was captured. | |||
| 147 | + | // Caller holds mu. | |||
| 148 | + | func (d *Daemon) purgeSupersededLocked(epoch uint64) bool { | |||
| 149 | + | return d.closing || d.purgeAdmissionClosing || d.purgeEpoch != epoch | |||
| 150 | + | } | |||
| 151 | + | ||||
| 152 | + | // purgeSuperseded is purgeSupersededLocked without the caller-held lock. | |||
| 153 | + | func (d *Daemon) purgeSuperseded(epoch uint64) bool { | |||
| 154 | + | d.mu.Lock() | |||
| 155 | + | defer d.mu.Unlock() | |||
| 156 | + | return d.purgeSupersededLocked(epoch) | |||
| 157 | + | } | |||
| 158 | + | ||||
| 159 | + | // purgeSupersededForShutdown reports whether an explicit daemon shutdown has | |||
| 160 | + | // begun. A KillAll purge defers to it so explicit stop semantics (preserving | |||
| 161 | + | // live sessions as stopped authority) are never overwritten by a concurrent | |||
| 162 | + | // purge. | |||
| 163 | + | func (d *Daemon) purgeSupersededForShutdown() bool { | |||
| 164 | + | d.mu.Lock() | |||
| 165 | + | defer d.mu.Unlock() | |||
| 166 | + | return d.closing | |||
| 167 | + | } | |||
| 168 | + | ||||
| 169 | + | // purgeAdmissionChangeLocked returns the channel closed when either purge | |||
| 170 | + | // admission field changes. Caller holds mu. | |||
| 171 | + | func (d *Daemon) purgeAdmissionChangeLocked() chan struct{} { | |||
| 172 | + | if d.purgeAdmissionChanged == nil { | |||
| 173 | + | d.purgeAdmissionChanged = make(chan struct{}) | |||
| 174 | + | } | |||
| 175 | + | return d.purgeAdmissionChanged | |||
| 176 | + | } | |||
| 177 | + | ||||
| 178 | + | // signalPurgeAdmissionChangedLocked wakes every admission waiter. Caller holds | |||
| 179 | + | // mu. | |||
| 180 | + | func (d *Daemon) signalPurgeAdmissionChangedLocked() { | |||
| 181 | + | close(d.purgeAdmissionChangeLocked()) | |||
| 182 | + | d.purgeAdmissionChanged = make(chan struct{}) | |||
| 183 | + | } | |||
| 184 | + | ||||
| 185 | + | // purgeRecordClass identifies which lifecycle record class a failed purge unit | |||
| 186 | + | // belongs to, so callers can report a typed partial failure instead of one | |||
| 187 | + | // opaque error. | |||
| 188 | + | type purgeRecordClass uint8 | |||
| 189 | + | ||||
| 190 | + | const ( | |||
| 191 | + | purgeRecordLive purgeRecordClass = iota + 1 | |||
| 192 | + | purgeRecordStopped | |||
| 193 | + | purgeRecordBroken | |||
| 194 | + | purgeRecordDurable | |||
| 195 | + | ) | |||
| 196 | + | ||||
| 197 | + | func (c purgeRecordClass) String() string { | |||
| 198 | + | switch c { | |||
| 199 | + | case purgeRecordLive: | |||
| 200 | + | return "live" | |||
| 201 | + | case purgeRecordStopped: | |||
| 202 | + | return "stopped" | |||
| 203 | + | case purgeRecordBroken: | |||
| 204 | + | return "broken" | |||
| 205 | + | case purgeRecordDurable: | |||
| 206 | + | return "durable" | |||
| 207 | + | default: | |||
| 208 | + | return "unknown" | |||
| 209 | + | } | |||
| 210 | + | } | |||
| 211 | + | ||||
| 212 | + | // inactivePurgeClass resolves the record class of a captured stopped/broken | |||
| 213 | + | // registry entry. | |||
| 214 | + | func inactivePurgeClass(entry inactiveSession) purgeRecordClass { | |||
| 215 | + | if entry.broken() { | |||
| 216 | + | return purgeRecordBroken | |||
| 217 | + | } | |||
| 218 | + | return purgeRecordStopped | |||
| 219 | + | } | |||
| 220 | + | ||||
| 221 | + | // purgeFailure is one structured partial purge failure: the record class, the | |||
| 222 | + | // affected name when known, and the underlying error. | |||
| 223 | + | type purgeFailure struct { | |||
| 224 | + | Class purgeRecordClass | |||
| 225 | + | Name string | |||
| 226 | + | Err error | |||
| 227 | + | } | |||
| 228 | + | ||||
| 229 | + | func (f purgeFailure) Error() string { | |||
| 230 | + | if f.Name == "" { | |||
| 231 | + | return fmt.Sprintf("%s: %v", f.Class, f.Err) | |||
| 232 | + | } | |||
| 233 | + | return fmt.Sprintf("%s %q: %v", f.Class, f.Name, f.Err) | |||
| 234 | + | } | |||
| 235 | + | ||||
| 236 | + | func (f purgeFailure) Unwrap() error { return f.Err } | |||
| 237 | + | ||||
| 238 | + | // purgeResult reports every record that could not be fully purged. An empty | |||
| 239 | + | // Failures slice means the purge removed the entire exact lifecycle set. A | |||
| 240 | + | // superseded result means an explicit daemon shutdown won the race: the purge | |||
| 241 | + | // deferred and removed nothing, or removed only the units whose teardown it had | |||
| 242 | + | // already owned when the stop began. A drain timeout result means an admitted | |||
| 243 | + | // transition refused to drain within the bounded admission budget, so the purge | |||
| 244 | + | // removed nothing and reopened admission. A typed live failure means a live unit | |||
| 245 | + | // was left untouched because the shared live-teardown budget expired before its | |||
| 246 | + | // destructive boundary or because its participants kept changing. A typed | |||
| 247 | + | // stopped/broken failure means the bounded stopped/broken sweep did not finish | |||
| 248 | + | // the record's durable purge within its own budget. | |||
| 249 | + | type purgeResult struct { | |||
| 250 | + | Failures []purgeFailure | |||
| 251 | + | superseded bool | |||
| 252 | + | drainTimedOut bool | |||
| 253 | + | } | |||
| 254 | + | ||||
| 255 | + | func (r purgeResult) Failed() bool { return len(r.Failures) > 0 || r.drainTimedOut } | |||
| 256 | + | ||||
| 257 | + | // Superseded reports that an explicit daemon shutdown owned the lifecycle set | |||
| 258 | + | // before this purge could finish destroying it, so the purge deferred to it. | |||
| 259 | + | // Any Failures it carries describe units the stop still owns; a superseded | |||
| 260 | + | // result never means the purge silently reported success. | |||
| 261 | + | func (r purgeResult) Superseded() bool { return r.superseded } | |||
| 262 | + | ||||
| 263 | + | // purgeSummaryFailureLimit bounds the wire-safe failure detail. The result may | |||
| 264 | + | // hold arbitrarily many failures (one per record), but a control response must | |||
| 265 | + | // stay bounded: report the total count and only the first N failures, then note | |||
| 266 | + | // how many were omitted. | |||
| 267 | + | const purgeSummaryFailureLimit = 5 | |||
| 268 | + | ||||
| 269 | + | func (r purgeResult) summary() string { | |||
| 270 | + | if r.drainTimedOut { | |||
| 271 | + | return "session purge aborted: purge admission did not drain within the bounded timeout; no records were removed" | |||
| 272 | + | } | |||
| 273 | + | if !r.Failed() { | |||
| 274 | + | return "" | |||
| 275 | + | } | |||
| 276 | + | shown := min(len(r.Failures), purgeSummaryFailureLimit) | |||
| 277 | + | parts := make([]string, 0, shown) | |||
| 278 | + | for _, failure := range r.Failures[:shown] { | |||
| 279 | + | parts = append(parts, failure.Error()) | |||
| 280 | + | } | |||
| 281 | + | if omitted := len(r.Failures) - shown; omitted > 0 { | |||
| 282 | + | return fmt.Sprintf("%d record(s) could not be purged; first %d: %s; %d more omitted", len(r.Failures), shown, strings.Join(parts, "; "), omitted) | |||
| 283 | + | } | |||
| 284 | + | return fmt.Sprintf("%d record(s) could not be purged: %s", len(r.Failures), strings.Join(parts, "; ")) | |||
| 285 | + | } | |||
| 286 | + | ||||
| 287 | + | // purgeAllSessions is the KillAll operation: it removes every live session and | |||
| 288 | + | // every stopped/broken/durable record while leaving the daemon active. It does | |||
| 289 | + | // not set closing, close move admission, cancel daemon-wide pane/process | |||
| 290 | + | // contexts, close done, or consume shutdown signals, and it never makes Serve | |||
| 291 | + | // return. | |||
| 292 | + | // | |||
| 293 | + | // Linearization: beginPurgeAdmission closes the transient admission gate and | |||
| 294 | + | // drains in-flight create/restore/move reservations, so the exact lifecycle set | |||
| 295 | + | // captured below cannot be mutated by a concurrent transition. Purge then runs | |||
| 296 | + | // against that exact set, and endPurgeAdmission reopens admission for later | |||
| 297 | + | // creations, which are ordered after the purge. | |||
| 298 | + | // | |||
| 299 | + | // Budgets: the admission drain, the stopped/broken durable sweep, and | |||
| 300 | + | // live-session teardown spend separate bounded budgets, each matching the | |||
| 301 | + | // daemon shutdown budget. A transition that ignores | |||
| 302 | + | // cancellation and burns the whole drain budget therefore cannot leave teardown | |||
| 303 | + | // with no time, and teardown cannot extend the drain. A repository deletion | |||
| 304 | + | // that ignores cancellation cannot extend the stopped/broken sweep past its own | |||
| 305 | + | // budget either. The live budget is armed | |||
| 306 | + | // when the live phase begins and is shared by every live unit: once it expires, | |||
| 307 | + | // each remaining live unit is skipped and reported as a typed live failure | |||
| 308 | + | // rather than reported as purged. Every unit keeps its own accurate name. | |||
| 309 | + | func (d *Daemon) purgeAllSessions(reason uint8) purgeResult { | |||
| 310 | + | d.purgeAllMu.Lock() | |||
| 311 | + | defer d.purgeAllMu.Unlock() | |||
| 312 | + | ||||
| 313 | + | // One deadline bounds the drain of every admitted transition. It is stopped | |||
| 314 | + | // before the destructive phase so teardown spends a fresh budget. | |||
| 315 | + | admissionDeadline := newSnapshotShutdownDeadline(d.clock) | |||
| 316 | + | defer admissionDeadline.stop() | |||
| 317 | + | ||||
| 318 | + | ctx := d.serveCtx | |||
| 319 | + | if ctx == nil { | |||
| 320 | + | ctx = context.Background() | |||
| 321 | + | } | |||
| 322 | + | switch d.beginPurgeAdmission(ctx, admissionDeadline) { | |||
| 323 | + | case purgeAdmissionSuperseded: | |||
| 324 | + | d.log.Info("session purge superseded before admission", "reason", reason) | |||
| 325 | + | d.endPurgeAdmission() | |||
| 326 | + | return purgeResult{superseded: true} | |||
| 327 | + | case purgeAdmissionTimedOut: | |||
| 328 | + | d.log.Warn("session purge aborted: purge admission did not drain", "reason", reason) | |||
| 329 | + | d.endPurgeAdmission() | |||
| 330 | + | return purgeResult{drainTimedOut: true} | |||
| 331 | + | } | |||
| 332 | + | admissionDeadline.stop() | |||
| 333 | + | defer d.endPurgeAdmission() | |||
| 334 | + | ||||
| 335 | + | d.mu.Lock() | |||
| 336 | + | // A shutdown that began after admission was granted still wins: no record | |||
| 337 | + | // has been removed yet, so defer without touching the preserved set. | |||
| 338 | + | if d.closing { | |||
| 339 | + | d.mu.Unlock() | |||
| 340 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "pre-snapshot") | |||
| 341 | + | return purgeResult{superseded: true} | |||
| 342 | + | } | |||
| 343 | + | d.purgeAllParkingLocked() | |||
| 344 | + | parkedRetirements := d.purgeAllParkedLocked() | |||
| 345 | + | d.purgeAllSuspendedLocked() | |||
| 346 | + | live := d.sessionsSnapshotLocked() | |||
| 347 | + | inactive := make([]inactiveSession, 0, len(d.inactive)) | |||
| 348 | + | exact := make(map[string]struct{}, len(live)+len(d.inactive)) | |||
| 349 | + | for _, entry := range d.inactive { | |||
| 350 | + | inactive = append(inactive, entry) | |||
| 351 | + | exact[entry.name] = struct{}{} | |||
| 352 | + | } | |||
| 353 | + | for _, sess := range live { | |||
| 354 | + | sess.mu.Lock() | |||
| 355 | + | exact[sess.name] = struct{}{} | |||
| 356 | + | sess.mu.Unlock() | |||
| 357 | + | } | |||
| 358 | + | d.mu.Unlock() | |||
| 359 | + | if d.afterPurgeAdmission != nil { | |||
| 360 | + | d.afterPurgeAdmission() | |||
| 361 | + | } | |||
| 362 | + | d.finishParkedAttachmentRetirements(parkedRetirements) | |||
| 363 | + | sort.Slice(inactive, func(i, j int) bool { return inactive[i].name < inactive[j].name }) | |||
| 364 | + | ||||
| 365 | + | // A shutdown that began while the snapshot was taken still wins: no record | |||
| 366 | + | // has been removed yet, so defer without touching the preserved set. | |||
| 367 | + | if d.purgeSupersededForShutdown() { | |||
| 368 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "pre-destructive") | |||
| 369 | + | return purgeResult{superseded: true} | |||
| 370 | + | } | |||
| 371 | + | ||||
| 372 | + | result := purgeResult{} | |||
| 373 | + | d.log.Info("session purge begin", "reason", reason, "live_sessions", len(live), "inactive_sessions", len(inactive)) | |||
| 374 | + | ||||
| 375 | + | // Stopped authority first (mirroring the shutdown purge order), then live | |||
| 376 | + | // sessions. Each unit is independent: one failure never cancels a sibling. | |||
| 377 | + | // Explicit stop wins per unit: a shutdown that begins mid-purge stops further | |||
| 378 | + | // removals so it can preserve every not-yet-purged lifecycle as stopped | |||
| 379 | + | // authority. A unit the purge already owns (its teardown is in progress) | |||
| 380 | + | // keeps purge authority and completes as a purge; only units not yet started | |||
| 381 | + | // are handed back to the stop. | |||
| 382 | + | // | |||
| 383 | + | // The stopped/broken sweep spends its own deadline and context, separate from | |||
| 384 | + | // the admission drain and the live-teardown budget. A repository deletion | |||
| 385 | + | // that ignores cancellation therefore cannot extend the purge: at the budget | |||
| 386 | + | // every record the sweep did not finish is still stopped/broken authority and | |||
| 387 | + | // is reported as a typed failure, and admission is released promptly. | |||
| 388 | + | if len(inactive) > 0 { | |||
| 389 | + | stoppedDeadline := newSnapshotShutdownDeadline(d.clock) | |||
| 390 | + | defer stoppedDeadline.stop() | |||
| 391 | + | stoppedCtx, cancelStopped := snapshotStopContext(stoppedDeadline) | |||
| 392 | + | defer cancelStopped() | |||
| 393 | + | sweep := d.purgeInactiveSweep(inactive, stoppedCtx, stoppedDeadline) | |||
| 394 | + | result.Failures = append(result.Failures, sweep.failures...) | |||
| 395 | + | if sweep.superseded { | |||
| 396 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "stopped") | |||
| 397 | + | return purgeResult{Failures: result.Failures, superseded: true} | |||
| 398 | + | } | |||
| 399 | + | } | |||
| 400 | + | // The live phase gets its own bounded budget, armed here so neither the | |||
| 401 | + | // admission drain nor the stopped-record purge consumes it. It is shared by | |||
| 402 | + | // every live unit. | |||
| 403 | + | destructiveDeadline := newSnapshotShutdownDeadline(d.clock) | |||
| 404 | + | defer destructiveDeadline.stop() | |||
| 405 | + | for _, sess := range live { | |||
| 406 | + | if d.purgeSupersededForShutdown() { | |||
| 407 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "live") | |||
| 408 | + | return purgeResult{Failures: result.Failures, superseded: true} | |||
| 409 | + | } | |||
| 410 | + | sess.mu.Lock() | |||
| 411 | + | name := sess.name | |||
| 412 | + | sess.mu.Unlock() | |||
| 413 | + | if d.beforePurgeLiveSessionKill != nil { | |||
| 414 | + | d.beforePurgeLiveSessionKill(sess) | |||
| 415 | + | } | |||
| 416 | + | sess.stopInMemoryLifecycle() | |||
| 417 | + | err := d.purgeLiveSession(sess, reason, destructiveDeadline) | |||
| 418 | + | if err == nil { | |||
| 419 | + | // A stop that wins this unit after the precheck above makes | |||
| 420 | + | // purgeLiveSession a no-op: the stop already removed the session and | |||
| 421 | + | // now owns it. Recheck so the purge never counts a stop-owned unit as a | |||
| 422 | + | // successful purge. A unit whose teardown the purge already owned keeps | |||
| 423 | + | // purge authority and is reported as removed, but the remaining units | |||
| 424 | + | // still belong to the stop. | |||
| 425 | + | if d.purgeSupersededForShutdown() { | |||
| 426 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "live-owned") | |||
| 427 | + | return purgeResult{Failures: result.Failures, superseded: true} | |||
| 428 | + | } | |||
| 429 | + | continue | |||
| 430 | + | } | |||
| 431 | + | if d.purgeSupersededForShutdown() && | |||
| 432 | + | (errors.Is(err, errSessionKillDeadline) || errors.Is(err, errSessionKillParticipantsChanged)) { | |||
| 433 | + | // The explicit stop owns the remaining lifecycles: either the live budget | |||
| 434 | + | // expired as it began, or a participant change aborted a kill that the | |||
| 435 | + | // stop now classifies as preservation rather than a purge failure. | |||
| 436 | + | d.log.Info("session purge superseded by daemon shutdown", "reason", reason, "phase", "live-deadline") | |||
| 437 | + | return purgeResult{Failures: result.Failures, superseded: true} | |||
| 438 | + | } | |||
| 439 | + | result.Failures = append(result.Failures, purgeFailure{Class: purgeRecordLive, Name: name, Err: err}) | |||
| 440 | + | d.log.Error("purging live session failed", "err", err, "session", name) | |||
| 441 | + | } | |||
| 442 | + | result.Failures = append(result.Failures, d.purgeDurableResidue(exact, result.Failures)...) | |||
| 443 | + | sort.SliceStable(result.Failures, func(i, j int) bool { | |||
| 444 | + | if result.Failures[i].Class != result.Failures[j].Class { | |||
| 445 | + | return result.Failures[i].Class < result.Failures[j].Class | |||
| 446 | + | } | |||
| 447 | + | return result.Failures[i].Name < result.Failures[j].Name | |||
| 448 | + | }) | |||
| 449 | + | if result.Failed() { | |||
| 450 | + | d.log.Warn("session purge incomplete", "reason", reason, "failures", len(result.Failures)) | |||
| 451 | + | } | |||
| 452 | + | return result | |||
| 453 | + | } | |||
| 454 | + | ||||
| 455 | + | // purgeDurableResidue removes durable records that have no live/stopped/broken | |||
| 456 | + | // registry entry, so a KillAll still accounts for every catalogue record. A | |||
| 457 | + | // record already represented by a reported live/stopped/broken failure is that | |||
| 458 | + | // failure's durable residue and is not double-reported here. | |||
| 459 | + | func (d *Daemon) purgeDurableResidue(exact map[string]struct{}, existing []purgeFailure) []purgeFailure { | |||
| 460 | + | if !d.persistEnabled || d.catalogue == nil { | |||
| 461 | + | return nil | |||
| 462 | + | } | |||
| 463 | + | records, err := d.catalogue.Records() | |||
| 464 | + | if err != nil { | |||
| 465 | + | return []purgeFailure{{Class: purgeRecordDurable, Err: err}} | |||
| 466 | + | } | |||
| 467 | + | reported := make(map[string]struct{}, len(existing)) | |||
| 468 | + | for _, failure := range existing { | |||
| 469 | + | reported[failure.Name] = struct{}{} | |||
| 470 | + | } | |||
| 471 | + | var out []purgeFailure | |||
| 472 | + | for _, record := range records { | |||
| 473 | + | if _, accounted := exact[record.Name]; accounted { | |||
| 474 | + | continue | |||
| 475 | + | } | |||
| 476 | + | if _, already := reported[record.Name]; already { | |||
| 477 | + | continue | |||
| 478 | + | } | |||
| 479 | + | if err := d.purgeDurableRecord(record); err != nil { | |||
| 480 | + | out = append(out, purgeFailure{Class: purgeRecordDurable, Name: record.Name, Err: err}) | |||
| 481 | + | } | |||
| 482 | + | } | |||
| 483 | + | return out | |||
| 484 | + | } | |||
| 485 | + | ||||
| 486 | + | // errSessionPurgeSweepDeadline reports that the stopped/broken sweep's own | |||
| 487 | + | // bounded budget expired before a record's durable purge finished. The record | |||
| 488 | + | // keeps its stopped/broken authority and is reported as a typed failure rather | |||
| 489 | + | // than silently counted as purged. | |||
| 490 | + | var errSessionPurgeSweepDeadline = errors.New("stopped session purge sweep deadline exceeded") | |||
| 491 | + | ||||
| 492 | + | // errSessionPurgeDeletePending reports that a purge-owned live unit's exact | |||
| 493 | + | // durable delete did not finish within the shared live-teardown budget. The | |||
| 494 | + | // live identity is already removed and its stopped authority still reserves the | |||
| 495 | + | // name; a detached worker finishes the fenced DeleteExact and removes that | |||
| 496 | + | // authority once the repository returns. It is a typed failure so a bounded | |||
| 497 | + | // purge never reports a skipped durable delete as success, and a repository | |||
| 498 | + | // delete that ignores cancellation can never hold purgeAllMu, the admission | |||
| 499 | + | // gate, or control responses. | |||
| 500 | + | var errSessionPurgeDeletePending = errors.New("session durable purge pending") | |||
| 501 | + | ||||
| 502 | + | // finishLiveDurablePurge runs the exact durable delete for a live unit the purge | |||
| 503 | + | // already removed from the live registry. With a bounded deadline it runs the | |||
| 504 | + | // delete on a detached worker: the caller returns errSessionPurgeDeletePending | |||
| 505 | + | // at the deadline while the worker finishes and removes the matching stopped | |||
| 506 | + | // authority. That removal is fenced by the captured name, incarnation, and | |||
| 507 | + | // createdAt, so a same-name or re-incarnated replacement is never touched. A | |||
| 508 | + | // nil deadline (a command-path kill) keeps the previous synchronous behavior. | |||
| 509 | + | func (d *Daemon) finishLiveDurablePurge(deadline *snapshotShutdownDeadline, record domain.CatalogueRecord) error { | |||
| 510 | + | expected := inactiveSession{name: record.Name, createdAt: record.CreatedAt, incarnation: record.IncarnationID, purging: true} | |||
| 511 | + | finish := func() error { | |||
| 512 | + | if err := d.finishSnapshotPurge(d.serveCtx, record.Name, record.IncarnationID, record.CreatedAt); err != nil { | |||
| 513 | + | return err | |||
| 514 | + | } | |||
| 515 | + | d.mu.Lock() | |||
| 516 | + | if stopped, ok := d.inactive[record.Name]; ok && stopped.purging && stopped.sameLifecycle(expected) { | |||
| 517 | + | delete(d.inactive, record.Name) | |||
| 518 | + | } | |||
| 519 | + | d.mu.Unlock() | |||
| 520 | + | d.reconcileAllRouteHistories() | |||
| 521 | + | return nil | |||
| 522 | + | } | |||
| 523 | + | if deadline == nil { | |||
| 524 | + | return finish() | |||
| 525 | + | } | |||
| 526 | + | done := make(chan error, 1) | |||
| 527 | + | go func() { done <- finish() }() | |||
| 528 | + | select { | |||
| 529 | + | case err := <-done: | |||
| 530 | + | return err | |||
| 531 | + | case <-deadline.Done(): | |||
| 532 | + | // Recheck completion so a delete that finished as the budget expired is | |||
| 533 | + | // reported as success rather than pending. | |||
| 534 | + | select { | |||
| 535 | + | case err := <-done: | |||
| 536 | + | return err | |||
| 537 | + | default: | |||
| 538 | + | // The caller already reported a typed pending failure and stopped | |||
| 539 | + | // waiting. Wait for the detached delete on its own goroutine so a late | |||
| 540 | + | // failure is still recorded with bounded, non-sensitive fields (record | |||
| 541 | + | // class, incarnation, created-time) instead of the user-chosen session | |||
| 542 | + | // name. | |||
| 543 | + | go func() { | |||
| 544 | + | if err := <-done; err != nil { | |||
| 545 | + | d.log.Warn("detached session purge delete failed after deadline", | |||
| 546 | + | "class", purgeRecordLive, "incarnation", record.IncarnationID, "created_at", record.CreatedAt, "err", err) | |||
| 547 | + | } | |||
| 548 | + | }() | |||
| 549 | + | return errSessionPurgeDeletePending | |||
| 550 | + | } | |||
| 551 | + | } | |||
| 552 | + | } | |||
| 553 | + | ||||
| 554 | + | // purgeInactiveSweepResult reports the bounded stopped/broken sweep outcome. | |||
| 555 | + | // failures holds every record the purge could not finish; superseded means an | |||
| 556 | + | // explicit stop began mid-sweep and owns the remaining records. | |||
| 557 | + | type purgeInactiveSweepResult struct { | |||
| 558 | + | failures []purgeFailure | |||
| 559 | + | superseded bool | |||
| 560 | + | } | |||
| 561 | + | ||||
| 562 | + | // purgeInactiveSweep removes the captured stopped/broken records under its own | |||
| 563 | + | // deadline and context, separate from the admission drain and the | |||
| 564 | + | // live-teardown budget. Because a repository deletion may ignore cancellation, | |||
| 565 | + | // the sweep runs on a detached worker and the caller returns at the deadline: | |||
| 566 | + | // every record the worker did not finish keeps its stopped/broken authority and | |||
| 567 | + | // is reported as a typed failure, so a bounded purge releases admission | |||
| 568 | + | // promptly instead of waiting out an uncooperative delete. The detached worker | |||
| 569 | + | // stops contributing to the returned result at that point; its registry removal | |||
| 570 | + | // stays lifecycle-fenced, so a late completion cannot delete a same-name | |||
| 571 | + | // replacement. | |||
| 572 | + | func (d *Daemon) purgeInactiveSweep(inactive []inactiveSession, ctx context.Context, deadline *snapshotShutdownDeadline) purgeInactiveSweepResult { | |||
| 573 | + | var ( | |||
| 574 | + | mu sync.Mutex | |||
| 575 | + | failures []purgeFailure | |||
| 576 | + | completed int | |||
| 577 | + | superseded bool | |||
| 578 | + | detached bool | |||
| 579 | + | ) | |||
| 580 | + | sweepDone := make(chan struct{}) | |||
| 581 | + | go func() { | |||
| 582 | + | defer close(sweepDone) | |||
| 583 | + | for _, entry := range inactive { | |||
| 584 | + | mu.Lock() | |||
| 585 | + | abandoned := detached | |||
| 586 | + | mu.Unlock() | |||
| 587 | + | if abandoned { | |||
| 588 | + | return | |||
| 589 | + | } | |||
| 590 | + | if d.purgeSupersededForShutdown() { | |||
| 591 | + | mu.Lock() | |||
| 592 | + | superseded = true | |||
| 593 | + | mu.Unlock() | |||
| 594 | + | return | |||
| 595 | + | } | |||
| 596 | + | class := inactivePurgeClass(entry) | |||
| 597 | + | err := d.retryStoppedPurgeContextExact(ctx, entry.name, entry.incarnation, &entry.createdAt) | |||
| 598 | + | mu.Lock() | |||
| 599 | + | if detached { | |||
| 600 | + | // The bounded caller already reported this unit; a late result must | |||
| 601 | + | // not mutate the returned result. | |||
| 602 | + | mu.Unlock() | |||
| 603 | + | return | |||
| 604 | + | } | |||
| 605 | + | if err != nil { | |||
| 606 | + | failures = append(failures, purgeFailure{Class: class, Name: entry.name, Err: err}) | |||
| 607 | + | d.log.Error("purging stopped session failed", "err", err, "session", entry.name) | |||
| 608 | + | } | |||
| 609 | + | completed++ | |||
| 610 | + | mu.Unlock() | |||
| 611 | + | } | |||
| 612 | + | }() | |||
| 613 | + | ||||
| 614 | + | select { | |||
| 615 | + | case <-sweepDone: | |||
| 616 | + | case <-deadline.Done(): | |||
| 617 | + | mu.Lock() | |||
| 618 | + | detached = true | |||
| 619 | + | done := completed | |||
| 620 | + | reported := append([]purgeFailure(nil), failures...) | |||
| 621 | + | mu.Unlock() | |||
| 622 | + | if d.purgeSupersededForShutdown() { | |||
| 623 | + | // An explicit stop that began as the budget expired owns the remaining | |||
| 624 | + | // records, so the sweep is supersession rather than a purge failure. | |||
| 625 | + | return purgeInactiveSweepResult{failures: reported, superseded: true} | |||
| 626 | + | } | |||
| 627 | + | for _, entry := range inactive[done:] { | |||
| 628 | + | reported = append(reported, purgeFailure{Class: inactivePurgeClass(entry), Name: entry.name, Err: errSessionPurgeSweepDeadline}) | |||
| 629 | + | d.log.Error("purging stopped session failed", "err", errSessionPurgeSweepDeadline, "session", entry.name) | |||
| 630 | + | } | |||
| 631 | + | return purgeInactiveSweepResult{failures: reported} | |||
| 632 | + | } | |||
| 633 | + | ||||
| 634 | + | mu.Lock() | |||
| 635 | + | defer mu.Unlock() | |||
| 636 | + | return purgeInactiveSweepResult{failures: failures, superseded: superseded} | |||
| 637 | + | } | |||
| 638 | + | ||||
| 639 | + | // purgeLiveSession removes one live session under the shared live-teardown | |||
| 640 | + | // budget. A post-freeze participant change aborts the kill before it touches the | |||
| 641 | + | // session. A session a racing path already removed owns nothing and counts as | |||
| 642 | + | // purged; an explicit stop that began meanwhile owns the rest and is never | |||
| 643 | + | // retried against; otherwise the purge retries once against a fresh snapshot, | |||
| 644 | + | // and a session whose participants changed again stays a typed live failure for | |||
| 645 | + | // the caller. | |||
| 646 | + | func (d *Daemon) purgeLiveSession(sess *session, reason uint8, deadline *snapshotShutdownDeadline) error { | |||
| 647 | + | err := d.killSessionWithSnapshotDeadline(sess, reason, true, deadline, nil) | |||
| 648 | + | if !errors.Is(err, errSessionKillParticipantsChanged) { | |||
| 649 | + | return err | |||
| 650 | + | } | |||
| 651 | + | if d.purgeSupersededForShutdown() { | |||
| 652 | + | // An explicit stop has begun: it owns the session's remaining lifecycle, so | |||
| 653 | + | // the purge must not retry against it. The caller classifies the abort as | |||
| 654 | + | // supersession instead of a live failure. | |||
| 655 | + | return err | |||
| 656 | + | } | |||
| 657 | + | if !d.sessionRegistered(sess) { | |||
| 658 | + | return nil | |||
| 659 | + | } | |||
| 660 | + | return d.killSessionWithSnapshotDeadline(sess, reason, true, deadline, nil) | |||
| 661 | + | } | |||
| 662 | + | ||||
| 663 | + | // sessionRegistered reports whether sess is still the exact live registry entry. | |||
| 664 | + | func (d *Daemon) sessionRegistered(sess *session) bool { | |||
| 665 | + | if sess == nil { | |||
| 666 | + | return false | |||
| 667 | + | } | |||
| 668 | + | d.mu.Lock() | |||
| 669 | + | defer d.mu.Unlock() | |||
| 670 | + | return d.sessions[sess.id] == sess | |||
| 671 | + | } | |||
| 672 | + | ||||
| 673 | + | func (d *Daemon) purgeDurableRecord(record domain.CatalogueRecord) error { | |||
| 674 | + | if !d.persistEnabled { | |||
| 675 | + | return nil | |||
| 676 | + | } | |||
| 677 | + | if d.recovery == nil { | |||
| 678 | + | return errors.New("durable session authority is not configured") | |||
| 679 | + | } | |||
| 680 | + | return d.finishSnapshotPurge(d.serveCtx, record.Name, record.IncarnationID, record.CreatedAt) | |||
| 681 | + | } | |||
A internal/usecase/daemon/purge_test.go +2386 -0 large diff (2386 lines changed): collapsed
M internal/usecase/daemon/remote_attach.go +22 -3
| @@ -40,6 +40,10 @@ func (d *Daemon) routeRemoteTargetWithContext(ctx context.Context, h protocol.He | |||
| 40 | 40 | d.mu.Unlock() | |
| 41 | 41 | return nil, nil, &protoErr{protocol.ErrServerShutdown, "daemon is shutting down"} | |
| 42 | 42 | } | |
| 43 | + | // Purge admission is already reserved by routeWithContext before this | |
| 44 | + | // dispatch, so an admitted remote target must not re-check the transient | |
| 45 | + | // gate: a purge that begins while this route is admitted waits for it and | |
| 46 | + | // must not reject it. | |
| 43 | 47 | if live := d.findByNameLocked(target.SessionName); live != nil { | |
| 44 | 48 | if live.incarnation != target.LifecycleID { | |
| 45 | 49 | d.mu.Unlock() | |
| @@ -127,12 +131,15 @@ func (d *Daemon) sendCommittedRouteIdentityForAttachment(effect *attachmentEffec | |||
| 127 | 131 | } | |
| 128 | 132 | ||
| 129 | 133 | func (d *Daemon) finishRouteAttach(sess *session, tr ports.ServerConnection, sz domain.Size, h protocol.Hello, routeCreated, purge bool) (*attachedClient, error) { | |
| 130 | - | ac, err := d.finishAttach(sess, tr, sz, h) | |
| 131 | - | if err != nil && routeCreated { | |
| 134 | + | ac, attachErr := d.finishAttach(sess, tr, sz, h) | |
| 135 | + | err := attachErr | |
| 136 | + | if attachErr != nil && routeCreated { | |
| 132 | 137 | if cleanupErr := d.killSessionIfEmpty(sess, protocol.ReasonSessionKilled, purge); cleanupErr != nil { | |
| 133 | 138 | err = errors.Join(err, cleanupErr) | |
| 134 | 139 | } | |
| 135 | - | return nil, err | |
| 140 | + | } | |
| 141 | + | if verdict, ok := d.routeAttachStopVerdict(attachErr); ok { | |
| 142 | + | return nil, verdict | |
| 136 | 143 | } | |
| 137 | 144 | if err == nil && routeCreated && ac != nil { | |
| 138 | 145 | ac.routeCreatedSession = true | |
| @@ -141,6 +148,18 @@ func (d *Daemon) finishRouteAttach(sess *session, tr ports.ServerConnection, sz | |||
| 141 | 148 | return ac, err | |
| 142 | 149 | } | |
| 143 | 150 | ||
| 151 | + | // routeAttachStopVerdict reports the authoritative explicit-stop outcome when an | |
| 152 | + | // attach failure is the leaked transition of a handshake the stop tore down. It | |
| 153 | + | // deliberately inspects only attachErr: the rollback cleanup error the caller | |
| 154 | + | // may join onto it is internal bookkeeping and must never decide the verdict, so | |
| 155 | + | // an unrelated attach failure is reported as-is even while the daemon stops. | |
| 156 | + | func (d *Daemon) routeAttachStopVerdict(attachErr error) (error, bool) { | |
| 157 | + | if attachErr == nil || !errors.Is(attachErr, errAttachmentTransition) || !d.purgeSupersededForShutdown() { | |
| 158 | + | return nil, false | |
| 159 | + | } | |
| 160 | + | return &protoErr{protocol.ErrServerShutdown, "daemon is shutting down"}, true | |
| 161 | + | } | |
| 162 | + | ||
| 144 | 163 | // remoteTargetMatchesSessionLocked validates the exact lifecycle and tab | |
| 145 | 164 | // selector while the daemon registry lock is held. Callers use this before | |
| 146 | 165 | // changing resume ownership so a stale target cannot partially claim a session. | |
| @@ -40,6 +40,10 @@ func (d *Daemon) routeRemoteTargetWithContext(ctx context.Context, h protocol.He | |||||
| 40 | d.mu.Unlock() | 40 | d.mu.Unlock() | ||
| 41 | return nil, nil, &protoErr{protocol.ErrServerShutdown, "daemon is shutting down"} | 41 | return nil, nil, &protoErr{protocol.ErrServerShutdown, "daemon is shutting down"} | ||
| 42 | } | 42 | } | ||
| 43 | + | // Purge admission is already reserved by routeWithContext before this | |||
| 44 | + | // dispatch, so an admitted remote target must not re-check the transient | |||
| 45 | + | // gate: a purge that begins while this route is admitted waits for it and | |||
| 46 | + | // must not reject it. | |||
| 43 | if live := d.findByNameLocked(target.SessionName); live != nil { | 47 | if live := d.findByNameLocked(target.SessionName); live != nil { | ||
| 44 | if live.incarnation != target.LifecycleID { | 48 | if live.incarnation != target.LifecycleID { | ||
| 45 | d.mu.Unlock() | 49 | d.mu.Unlock() | ||
| @@ -127,12 +131,15 @@ func (d *Daemon) sendCommittedRouteIdentityForAttachment(effect *attachmentEffec | |||||
| 127 | } | 131 | } | ||
| 128 | 132 | ||||
| 129 | func (d *Daemon) finishRouteAttach(sess *session, tr ports.ServerConnection, sz domain.Size, h protocol.Hello, routeCreated, purge bool) (*attachedClient, error) { | 133 | func (d *Daemon) finishRouteAttach(sess *session, tr ports.ServerConnection, sz domain.Size, h protocol.Hello, routeCreated, purge bool) (*attachedClient, error) { | ||
| 130 | - | ac, err := d.finishAttach(sess, tr, sz, h) | 134 | + | ac, attachErr := d.finishAttach(sess, tr, sz, h) |
| 131 | - | if err != nil && routeCreated { | 135 | + | err := attachErr |
| 136 | + | if attachErr != nil && routeCreated { | |||
| 132 | if cleanupErr := d.killSessionIfEmpty(sess, protocol.ReasonSessionKilled, purge); cleanupErr != nil { | 137 | if cleanupErr := d.killSessionIfEmpty(sess, protocol.ReasonSessionKilled, purge); cleanupErr != nil { | ||
| 133 | err = errors.Join(err, cleanupErr) | 138 | err = errors.Join(err, cleanupErr) | ||
| 134 | } | 139 | } | ||
| 135 | - | return nil, err | 140 | + | } |
| 141 | + | if verdict, ok := d.routeAttachStopVerdict(attachErr); ok { | |||
| 142 | + | return nil, verdict | |||
| 136 | } | 143 | } | ||
| 137 | if err == nil && routeCreated && ac != nil { | 144 | if err == nil && routeCreated && ac != nil { | ||
| 138 | ac.routeCreatedSession = true | 145 | ac.routeCreatedSession = true | ||
| @@ -141,6 +148,18 @@ func (d *Daemon) finishRouteAttach(sess *session, tr ports.ServerConnection, sz | |||||
| 141 | return ac, err | 148 | return ac, err | ||
| 142 | } | 149 | } | ||
| 143 | 150 | ||||
| 151 | + | // routeAttachStopVerdict reports the authoritative explicit-stop outcome when an | |||
| 152 | + | // attach failure is the leaked transition of a handshake the stop tore down. It | |||
| 153 | + | // deliberately inspects only attachErr: the rollback cleanup error the caller | |||
| 154 | + | // may join onto it is internal bookkeeping and must never decide the verdict, so | |||
| 155 | + | // an unrelated attach failure is reported as-is even while the daemon stops. | |||
| 156 | + | func (d *Daemon) routeAttachStopVerdict(attachErr error) (error, bool) { | |||
| 157 | + | if attachErr == nil || !errors.Is(attachErr, errAttachmentTransition) || !d.purgeSupersededForShutdown() { | |||
| 158 | + | return nil, false | |||
| 159 | + | } | |||
| 160 | + | return &protoErr{protocol.ErrServerShutdown, "daemon is shutting down"}, true | |||
| 161 | + | } | |||
| 162 | + | ||||
| 144 | // remoteTargetMatchesSessionLocked validates the exact lifecycle and tab | 163 | // remoteTargetMatchesSessionLocked validates the exact lifecycle and tab | ||
| 145 | // selector while the daemon registry lock is held. Callers use this before | 164 | // selector while the daemon registry lock is held. Callers use this before | ||
| 146 | // changing resume ownership so a stale target cannot partially claim a session. | 165 | // changing resume ownership so a stale target cannot partially claim a session. | ||
M internal/usecase/daemon/remote_monitor_lifecycle_test.go +29 -9
| @@ -156,32 +156,52 @@ func TestServeStartsMonitorWithoutWaitingForRemote(t *testing.T) { | |||
| 156 | 156 | } | |
| 157 | 157 | } | |
| 158 | 158 | ||
| 159 | - | // TestServeIdleExitWithMonitorRunning proves monitoring never counts as an | |
| 160 | - | // attachment: the last session exit still drives Serve to return, and the | |
| 161 | - | // monitor stops with the daemon. | |
| 162 | - | func TestServeIdleExitWithMonitorRunning(t *testing.T) { | |
| 159 | + | // TestServeSurvivesIdleExitWithMonitorRunning proves monitoring never counts as | |
| 160 | + | // an attachment and the last session exit no longer drives Serve to return: the | |
| 161 | + | // daemon keeps serving and the monitor keeps running across the empty | |
| 162 | + | // registry, and only explicit cancellation stops both. | |
| 163 | + | func TestServeSurvivesIdleExitWithMonitorRunning(t *testing.T) { | |
| 163 | 164 | monitor := &scriptRemoteMonitor{started: make(chan struct{}), stopped: make(chan struct{})} | |
| 164 | 165 | d, l, _, sends, releasePTY := newMonitorServeHarness(t, monitor) | |
| 166 | + | defer releasePTY() | |
| 165 | 167 | ||
| 168 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 169 | + | defer cancel() | |
| 166 | 170 | served := make(chan error, 1) | |
| 167 | - | go func() { served <- d.Serve(context.Background(), l) }() | |
| 171 | + | go func() { served <- d.Serve(ctx, l) }() | |
| 168 | 172 | ||
| 169 | 173 | awaitFrame(t, sends, "Welcome") | |
| 170 | 174 | awaitFrame(t, sends, "Output") | |
| 171 | 175 | ||
| 172 | - | // Child exits -> session removed -> registry empties -> Serve returns, | |
| 173 | - | // even though the monitor runner is still parked in its context. | |
| 176 | + | // Child exits -> session removed -> registry empties. Serve must keep | |
| 177 | + | // running and the monitor must stay parked in its context. | |
| 174 | 178 | releasePTY() | |
| 179 | + | det := awaitFrame(t, sends, "Detached") | |
| 180 | + | require.Equal(t, protocol.ReasonSessionKilled, decodeServerMessage(t, det).(protocol.Detached).Reason) | |
| 181 | + | require.Equal(t, 0, sessionCount(d)) | |
| 182 | + | select { | |
| 183 | + | case <-served: | |
| 184 | + | t.Fatal("Serve returned after the last session exited") | |
| 185 | + | default: | |
| 186 | + | } | |
| 187 | + | select { | |
| 188 | + | case <-monitor.stopped: | |
| 189 | + | t.Fatal("monitor stopped while the daemon was still serving") | |
| 190 | + | default: | |
| 191 | + | } | |
| 192 | + | ||
| 193 | + | // Only explicit cancellation ends the daemon. | |
| 194 | + | cancel() | |
| 175 | 195 | select { | |
| 176 | 196 | case err := <-served: | |
| 177 | 197 | require.NoError(t, err) | |
| 178 | 198 | case <-time.After(10 * time.Second): | |
| 179 | - | t.Fatal("Serve did not return after last session exited") | |
| 199 | + | t.Fatal("Serve did not return after context cancel") | |
| 180 | 200 | } | |
| 181 | 201 | select { | |
| 182 | 202 | case <-monitor.stopped: | |
| 183 | 203 | case <-time.After(5 * time.Second): | |
| 184 | - | t.Fatal("monitor runner did not stop after idle exit") | |
| 204 | + | t.Fatal("monitor runner did not stop after explicit shutdown") | |
| 185 | 205 | } | |
| 186 | 206 | } | |
| 187 | 207 | ||
| @@ -156,32 +156,52 @@ func TestServeStartsMonitorWithoutWaitingForRemote(t *testing.T) { | |||||
| 156 | } | 156 | } | ||
| 157 | } | 157 | } | ||
| 158 | 158 | ||||
| 159 | - | // TestServeIdleExitWithMonitorRunning proves monitoring never counts as an | 159 | + | // TestServeSurvivesIdleExitWithMonitorRunning proves monitoring never counts as |
| 160 | - | // attachment: the last session exit still drives Serve to return, and the | 160 | + | // an attachment and the last session exit no longer drives Serve to return: the |
| 161 | - | // monitor stops with the daemon. | 161 | + | // daemon keeps serving and the monitor keeps running across the empty |
| 162 | - | func TestServeIdleExitWithMonitorRunning(t *testing.T) { | 162 | + | // registry, and only explicit cancellation stops both. |
| 163 | + | func TestServeSurvivesIdleExitWithMonitorRunning(t *testing.T) { | |||
| 163 | monitor := &scriptRemoteMonitor{started: make(chan struct{}), stopped: make(chan struct{})} | 164 | monitor := &scriptRemoteMonitor{started: make(chan struct{}), stopped: make(chan struct{})} | ||
| 164 | d, l, _, sends, releasePTY := newMonitorServeHarness(t, monitor) | 165 | d, l, _, sends, releasePTY := newMonitorServeHarness(t, monitor) | ||
| 166 | + | defer releasePTY() | |||
| 165 | 167 | ||||
| 168 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 169 | + | defer cancel() | |||
| 166 | served := make(chan error, 1) | 170 | served := make(chan error, 1) | ||
| 167 | - | go func() { served <- d.Serve(context.Background(), l) }() | 171 | + | go func() { served <- d.Serve(ctx, l) }() |
| 168 | 172 | ||||
| 169 | awaitFrame(t, sends, "Welcome") | 173 | awaitFrame(t, sends, "Welcome") | ||
| 170 | awaitFrame(t, sends, "Output") | 174 | awaitFrame(t, sends, "Output") | ||
| 171 | 175 | ||||
| 172 | - | // Child exits -> session removed -> registry empties -> Serve returns, | 176 | + | // Child exits -> session removed -> registry empties. Serve must keep |
| 173 | - | // even though the monitor runner is still parked in its context. | 177 | + | // running and the monitor must stay parked in its context. |
| 174 | releasePTY() | 178 | releasePTY() | ||
| 179 | + | det := awaitFrame(t, sends, "Detached") | |||
| 180 | + | require.Equal(t, protocol.ReasonSessionKilled, decodeServerMessage(t, det).(protocol.Detached).Reason) | |||
| 181 | + | require.Equal(t, 0, sessionCount(d)) | |||
| 182 | + | select { | |||
| 183 | + | case <-served: | |||
| 184 | + | t.Fatal("Serve returned after the last session exited") | |||
| 185 | + | default: | |||
| 186 | + | } | |||
| 187 | + | select { | |||
| 188 | + | case <-monitor.stopped: | |||
| 189 | + | t.Fatal("monitor stopped while the daemon was still serving") | |||
| 190 | + | default: | |||
| 191 | + | } | |||
| 192 | + | ||||
| 193 | + | // Only explicit cancellation ends the daemon. | |||
| 194 | + | cancel() | |||
| 175 | select { | 195 | select { | ||
| 176 | case err := <-served: | 196 | case err := <-served: | ||
| 177 | require.NoError(t, err) | 197 | require.NoError(t, err) | ||
| 178 | case <-time.After(10 * time.Second): | 198 | case <-time.After(10 * time.Second): | ||
| 179 | - | t.Fatal("Serve did not return after last session exited") | 199 | + | t.Fatal("Serve did not return after context cancel") |
| 180 | } | 200 | } | ||
| 181 | select { | 201 | select { | ||
| 182 | case <-monitor.stopped: | 202 | case <-monitor.stopped: | ||
| 183 | case <-time.After(5 * time.Second): | 203 | case <-time.After(5 * time.Second): | ||
| 184 | - | t.Fatal("monitor runner did not stop after idle exit") | 204 | + | t.Fatal("monitor runner did not stop after explicit shutdown") |
| 185 | } | 205 | } | ||
| 186 | } | 206 | } | ||
| 187 | 207 | ||||
M internal/usecase/daemon/remote_route_test.go +119 -0
| @@ -2,6 +2,8 @@ package daemon | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | + | "errors" | |
| 6 | + | "fmt" | |
| 5 | 7 | "os" | |
| 6 | 8 | "testing" | |
| 7 | 9 | ||
| @@ -222,3 +224,120 @@ func TestRouteRemoteTargetRejectsSameNameReplacement(t *testing.T) { | |||
| 222 | 224 | require.ErrorAs(t, err, &protocolErr) | |
| 223 | 225 | require.Equal(t, protocol.ErrNoSuchTarget, protocolErr.code) | |
| 224 | 226 | } | |
| 227 | + | ||
| 228 | + | // TestRouteAttachStopVerdictKeysOnAttachError proves the supersession rewrite is | |
| 229 | + | // keyed on the attach failure: a cleanup error that happens to chain | |
| 230 | + | // errAttachmentTransition must not turn an unrelated attach failure into the | |
| 231 | + | // shutdown verdict, while a genuine attach-level transition under a stop still | |
| 232 | + | // is. | |
| 233 | + | func TestRouteAttachStopVerdictKeysOnAttachError(t *testing.T) { | |
| 234 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 235 | + | d.mu.Lock() | |
| 236 | + | d.closing = true | |
| 237 | + | d.mu.Unlock() | |
| 238 | + | t.Cleanup(func() { | |
| 239 | + | d.mu.Lock() | |
| 240 | + | d.closing = false | |
| 241 | + | d.mu.Unlock() | |
| 242 | + | }) | |
| 243 | + | ||
| 244 | + | attachErr := errors.New("route attach failed") | |
| 245 | + | cleanupErr := fmt.Errorf("route cleanup: %w", errAttachmentTransition) | |
| 246 | + | joined := errors.Join(attachErr, cleanupErr) | |
| 247 | + | // The joined rollback error really does carry the transition sentinel, which | |
| 248 | + | // is exactly what the old rewrite keyed on. | |
| 249 | + | require.ErrorIs(t, joined, errAttachmentTransition, "the synthetic join still carries the sentinel") | |
| 250 | + | verdict, ok := d.routeAttachStopVerdict(attachErr) | |
| 251 | + | require.False(t, ok, "a cleanup-only transition must not decide the verdict") | |
| 252 | + | require.Nil(t, verdict) | |
| 253 | + | ||
| 254 | + | verdict, ok = d.routeAttachStopVerdict(errAttachmentTransition) | |
| 255 | + | require.True(t, ok) | |
| 256 | + | var protocolErr *protoErr | |
| 257 | + | require.ErrorAs(t, verdict, &protocolErr) | |
| 258 | + | require.Equal(t, protocol.ErrServerShutdown, protocolErr.code) | |
| 259 | + | ||
| 260 | + | d.mu.Lock() | |
| 261 | + | d.closing = false | |
| 262 | + | d.mu.Unlock() | |
| 263 | + | _, ok = d.routeAttachStopVerdict(errAttachmentTransition) | |
| 264 | + | require.False(t, ok, "a transition without an explicit stop is not a shutdown verdict") | |
| 265 | + | } | |
| 266 | + | ||
| 267 | + | // TestFinishRouteAttachSupersessionReportsShutdownVerdict proves the reachable | |
| 268 | + | // supersession path: an explicit stop that removes the route's target during the | |
| 269 | + | // handshake leaves the attach a leaked transition, and the caller receives the | |
| 270 | + | // authoritative shutdown verdict rather than the internal sentinel. | |
| 271 | + | func TestFinishRouteAttachSupersessionReportsShutdownVerdict(t *testing.T) { | |
| 272 | + | pty, release := newBlockingPTY(t) | |
| 273 | + | defer release() | |
| 274 | + | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | |
| 275 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", defaultSize, terminalEnv{}, d.baseEnv) | |
| 276 | + | require.NoError(t, err) | |
| 277 | + | ||
| 278 | + | d.afterAttachmentEffectParticipantsSnapshotted = func(action string, _ []*attachedClient) { | |
| 279 | + | if action != "" { | |
| 280 | + | return | |
| 281 | + | } | |
| 282 | + | d.afterAttachmentEffectParticipantsSnapshotted = nil | |
| 283 | + | // The explicit stop wins the target in the freeze-to-publish window, so | |
| 284 | + | // the attach transition can no longer be published. | |
| 285 | + | d.mu.Lock() | |
| 286 | + | d.closing = true | |
| 287 | + | d.unregisterSessionLocked(sess) | |
| 288 | + | d.mu.Unlock() | |
| 289 | + | } | |
| 290 | + | t.Cleanup(func() { d.afterAttachmentEffectParticipantsSnapshotted = nil }) | |
| 291 | + | ||
| 292 | + | // The caller must hold d.mu on entry; finishRouteAttach releases it on both | |
| 293 | + | // success and error paths before returning. | |
| 294 | + | d.mu.Lock() | |
| 295 | + | _, err = d.finishRouteAttach(sess, &closeTrackingTransport{}, defaultSize, protocol.Hello{ | |
| 296 | + | Version: protocol.Version, Intent: protocol.IntentAttach, Name: "work", Size: defaultSize, | |
| 297 | + | }, true, false) | |
| 298 | + | ||
| 299 | + | var protocolErr *protoErr | |
| 300 | + | require.ErrorAs(t, err, &protocolErr) | |
| 301 | + | require.Equal(t, protocol.ErrServerShutdown, protocolErr.code) | |
| 302 | + | d.mu.Lock() | |
| 303 | + | _, retained := d.sessions[sess.id] | |
| 304 | + | d.mu.Unlock() | |
| 305 | + | require.False(t, retained, "the stop owns the target from this point") | |
| 306 | + | } | |
| 307 | + | ||
| 308 | + | // TestFinishRouteAttachKeepsUnrelatedAttachFailure proves the call site keeps an | |
| 309 | + | // attach failure that is not a leaked transition, even while the daemon stops and | |
| 310 | + | // the rollback cleanup aborts with its own sentinel: only the attach error may | |
| 311 | + | // decide the shutdown verdict. | |
| 312 | + | func TestFinishRouteAttachKeepsUnrelatedAttachFailure(t *testing.T) { | |
| 313 | + | d, sess, ac, _ := newManualSessionWithPTYs(t, newQuietPTY()) | |
| 314 | + | ||
| 315 | + | d.afterAttachmentEffectParticipantsSnapshotted = func(action string, _ []*attachedClient) { | |
| 316 | + | if action != "" { | |
| 317 | + | return | |
| 318 | + | } | |
| 319 | + | d.afterAttachmentEffectParticipantsSnapshotted = nil | |
| 320 | + | // The stop begins and a token resume invalidates the rollback's snapshot, | |
| 321 | + | // so the cleanup aborts with a sentinel instead of a clean no-op. | |
| 322 | + | d.mu.Lock() | |
| 323 | + | d.closing = true | |
| 324 | + | d.mu.Unlock() | |
| 325 | + | fresh := &closeTrackingTransport{} | |
| 326 | + | ac.replaceTransport(fresh) | |
| 327 | + | ac.installTestAttachmentCapability(sess.captureAttachmentCapability(ac, fresh)) | |
| 328 | + | } | |
| 329 | + | t.Cleanup(func() { d.afterAttachmentEffectParticipantsSnapshotted = nil }) | |
| 330 | + | ||
| 331 | + | missing := domain.RemoteSessionTarget{LifecycleID: sess.incarnation, SessionName: "work", LiveTabID: "missing-tab"} | |
| 332 | + | d.mu.Lock() | |
| 333 | + | _, err := d.finishRouteAttach(sess, &closeTrackingTransport{}, defaultSize, protocol.Hello{ | |
| 334 | + | Version: protocol.Version, Intent: protocol.IntentAttach, Name: "work", Size: defaultSize, | |
| 335 | + | RemoteTarget: &missing, EnvironmentPolicy: protocol.EnvironmentPolicyDaemonOwned, | |
| 336 | + | }, true, false) | |
| 337 | + | var protocolErr *protoErr | |
| 338 | + | require.ErrorAs(t, err, &protocolErr) | |
| 339 | + | require.Equal(t, protocol.ErrNoSuchTarget, protocolErr.code, "an unrelated attach failure must not become a shutdown verdict") | |
| 340 | + | d.mu.Lock() | |
| 341 | + | d.closing = false | |
| 342 | + | d.mu.Unlock() | |
| 343 | + | } | |
| @@ -2,6 +2,8 @@ package daemon | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "errors" | |||
| 6 | + | "fmt" | |||
| 5 | "os" | 7 | "os" | ||
| 6 | "testing" | 8 | "testing" | ||
| 7 | 9 | ||||
| @@ -222,3 +224,120 @@ func TestRouteRemoteTargetRejectsSameNameReplacement(t *testing.T) { | |||||
| 222 | require.ErrorAs(t, err, &protocolErr) | 224 | require.ErrorAs(t, err, &protocolErr) | ||
| 223 | require.Equal(t, protocol.ErrNoSuchTarget, protocolErr.code) | 225 | require.Equal(t, protocol.ErrNoSuchTarget, protocolErr.code) | ||
| 224 | } | 226 | } | ||
| 227 | + | ||||
| 228 | + | // TestRouteAttachStopVerdictKeysOnAttachError proves the supersession rewrite is | |||
| 229 | + | // keyed on the attach failure: a cleanup error that happens to chain | |||
| 230 | + | // errAttachmentTransition must not turn an unrelated attach failure into the | |||
| 231 | + | // shutdown verdict, while a genuine attach-level transition under a stop still | |||
| 232 | + | // is. | |||
| 233 | + | func TestRouteAttachStopVerdictKeysOnAttachError(t *testing.T) { | |||
| 234 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 235 | + | d.mu.Lock() | |||
| 236 | + | d.closing = true | |||
| 237 | + | d.mu.Unlock() | |||
| 238 | + | t.Cleanup(func() { | |||
| 239 | + | d.mu.Lock() | |||
| 240 | + | d.closing = false | |||
| 241 | + | d.mu.Unlock() | |||
| 242 | + | }) | |||
| 243 | + | ||||
| 244 | + | attachErr := errors.New("route attach failed") | |||
| 245 | + | cleanupErr := fmt.Errorf("route cleanup: %w", errAttachmentTransition) | |||
| 246 | + | joined := errors.Join(attachErr, cleanupErr) | |||
| 247 | + | // The joined rollback error really does carry the transition sentinel, which | |||
| 248 | + | // is exactly what the old rewrite keyed on. | |||
| 249 | + | require.ErrorIs(t, joined, errAttachmentTransition, "the synthetic join still carries the sentinel") | |||
| 250 | + | verdict, ok := d.routeAttachStopVerdict(attachErr) | |||
| 251 | + | require.False(t, ok, "a cleanup-only transition must not decide the verdict") | |||
| 252 | + | require.Nil(t, verdict) | |||
| 253 | + | ||||
| 254 | + | verdict, ok = d.routeAttachStopVerdict(errAttachmentTransition) | |||
| 255 | + | require.True(t, ok) | |||
| 256 | + | var protocolErr *protoErr | |||
| 257 | + | require.ErrorAs(t, verdict, &protocolErr) | |||
| 258 | + | require.Equal(t, protocol.ErrServerShutdown, protocolErr.code) | |||
| 259 | + | ||||
| 260 | + | d.mu.Lock() | |||
| 261 | + | d.closing = false | |||
| 262 | + | d.mu.Unlock() | |||
| 263 | + | _, ok = d.routeAttachStopVerdict(errAttachmentTransition) | |||
| 264 | + | require.False(t, ok, "a transition without an explicit stop is not a shutdown verdict") | |||
| 265 | + | } | |||
| 266 | + | ||||
| 267 | + | // TestFinishRouteAttachSupersessionReportsShutdownVerdict proves the reachable | |||
| 268 | + | // supersession path: an explicit stop that removes the route's target during the | |||
| 269 | + | // handshake leaves the attach a leaked transition, and the caller receives the | |||
| 270 | + | // authoritative shutdown verdict rather than the internal sentinel. | |||
| 271 | + | func TestFinishRouteAttachSupersessionReportsShutdownVerdict(t *testing.T) { | |||
| 272 | + | pty, release := newBlockingPTY(t) | |||
| 273 | + | defer release() | |||
| 274 | + | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | |||
| 275 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", defaultSize, terminalEnv{}, d.baseEnv) | |||
| 276 | + | require.NoError(t, err) | |||
| 277 | + | ||||
| 278 | + | d.afterAttachmentEffectParticipantsSnapshotted = func(action string, _ []*attachedClient) { | |||
| 279 | + | if action != "" { | |||
| 280 | + | return | |||
| 281 | + | } | |||
| 282 | + | d.afterAttachmentEffectParticipantsSnapshotted = nil | |||
| 283 | + | // The explicit stop wins the target in the freeze-to-publish window, so | |||
| 284 | + | // the attach transition can no longer be published. | |||
| 285 | + | d.mu.Lock() | |||
| 286 | + | d.closing = true | |||
| 287 | + | d.unregisterSessionLocked(sess) | |||
| 288 | + | d.mu.Unlock() | |||
| 289 | + | } | |||
| 290 | + | t.Cleanup(func() { d.afterAttachmentEffectParticipantsSnapshotted = nil }) | |||
| 291 | + | ||||
| 292 | + | // The caller must hold d.mu on entry; finishRouteAttach releases it on both | |||
| 293 | + | // success and error paths before returning. | |||
| 294 | + | d.mu.Lock() | |||
| 295 | + | _, err = d.finishRouteAttach(sess, &closeTrackingTransport{}, defaultSize, protocol.Hello{ | |||
| 296 | + | Version: protocol.Version, Intent: protocol.IntentAttach, Name: "work", Size: defaultSize, | |||
| 297 | + | }, true, false) | |||
| 298 | + | ||||
| 299 | + | var protocolErr *protoErr | |||
| 300 | + | require.ErrorAs(t, err, &protocolErr) | |||
| 301 | + | require.Equal(t, protocol.ErrServerShutdown, protocolErr.code) | |||
| 302 | + | d.mu.Lock() | |||
| 303 | + | _, retained := d.sessions[sess.id] | |||
| 304 | + | d.mu.Unlock() | |||
| 305 | + | require.False(t, retained, "the stop owns the target from this point") | |||
| 306 | + | } | |||
| 307 | + | ||||
| 308 | + | // TestFinishRouteAttachKeepsUnrelatedAttachFailure proves the call site keeps an | |||
| 309 | + | // attach failure that is not a leaked transition, even while the daemon stops and | |||
| 310 | + | // the rollback cleanup aborts with its own sentinel: only the attach error may | |||
| 311 | + | // decide the shutdown verdict. | |||
| 312 | + | func TestFinishRouteAttachKeepsUnrelatedAttachFailure(t *testing.T) { | |||
| 313 | + | d, sess, ac, _ := newManualSessionWithPTYs(t, newQuietPTY()) | |||
| 314 | + | ||||
| 315 | + | d.afterAttachmentEffectParticipantsSnapshotted = func(action string, _ []*attachedClient) { | |||
| 316 | + | if action != "" { | |||
| 317 | + | return | |||
| 318 | + | } | |||
| 319 | + | d.afterAttachmentEffectParticipantsSnapshotted = nil | |||
| 320 | + | // The stop begins and a token resume invalidates the rollback's snapshot, | |||
| 321 | + | // so the cleanup aborts with a sentinel instead of a clean no-op. | |||
| 322 | + | d.mu.Lock() | |||
| 323 | + | d.closing = true | |||
| 324 | + | d.mu.Unlock() | |||
| 325 | + | fresh := &closeTrackingTransport{} | |||
| 326 | + | ac.replaceTransport(fresh) | |||
| 327 | + | ac.installTestAttachmentCapability(sess.captureAttachmentCapability(ac, fresh)) | |||
| 328 | + | } | |||
| 329 | + | t.Cleanup(func() { d.afterAttachmentEffectParticipantsSnapshotted = nil }) | |||
| 330 | + | ||||
| 331 | + | missing := domain.RemoteSessionTarget{LifecycleID: sess.incarnation, SessionName: "work", LiveTabID: "missing-tab"} | |||
| 332 | + | d.mu.Lock() | |||
| 333 | + | _, err := d.finishRouteAttach(sess, &closeTrackingTransport{}, defaultSize, protocol.Hello{ | |||
| 334 | + | Version: protocol.Version, Intent: protocol.IntentAttach, Name: "work", Size: defaultSize, | |||
| 335 | + | RemoteTarget: &missing, EnvironmentPolicy: protocol.EnvironmentPolicyDaemonOwned, | |||
| 336 | + | }, true, false) | |||
| 337 | + | var protocolErr *protoErr | |||
| 338 | + | require.ErrorAs(t, err, &protocolErr) | |||
| 339 | + | require.Equal(t, protocol.ErrNoSuchTarget, protocolErr.code, "an unrelated attach failure must not become a shutdown verdict") | |||
| 340 | + | d.mu.Lock() | |||
| 341 | + | d.closing = false | |||
| 342 | + | d.mu.Unlock() | |||
| 343 | + | } | |||
M internal/usecase/daemon/render_test.go +1 -1
| @@ -886,7 +886,7 @@ func TestResizeOrdersPTYBeforeScreen(t *testing.T) { | |||
| 886 | 886 | require.True(t, gotOutput.Load(), "resize forces a full redraw output") | |
| 887 | 887 | } | |
| 888 | 888 | ||
| 889 | - | // --- reader EOF -> registry-empty shutdown ---------------------------------- | |
| 889 | + | // --- failed client send teardown -------------------------------------------- | |
| 890 | 890 | ||
| 891 | 891 | func TestSendErrorKeepsEphemeralHeadless(t *testing.T) { | |
| 892 | 892 | p := portsmocks.NewMockPTY(t) | |
| @@ -886,7 +886,7 @@ func TestResizeOrdersPTYBeforeScreen(t *testing.T) { | |||||
| 886 | require.True(t, gotOutput.Load(), "resize forces a full redraw output") | 886 | require.True(t, gotOutput.Load(), "resize forces a full redraw output") | ||
| 887 | } | 887 | } | ||
| 888 | 888 | ||||
| 889 | - | // --- reader EOF -> registry-empty shutdown ---------------------------------- | 889 | + | // --- failed client send teardown -------------------------------------------- |
| 890 | 890 | ||||
| 891 | func TestSendErrorKeepsEphemeralHeadless(t *testing.T) { | 891 | func TestSendErrorKeepsEphemeralHeadless(t *testing.T) { | ||
| 892 | p := portsmocks.NewMockPTY(t) | 892 | p := portsmocks.NewMockPTY(t) | ||
M internal/usecase/daemon/session.go +159 -27
| @@ -470,6 +470,10 @@ func (d *Daemon) createSessionAndSwitch(from *session, ac *attachedClient, name | |||
| 470 | 470 | d.mu.Unlock() | |
| 471 | 471 | return errors.New("daemon is shutting down") | |
| 472 | 472 | } | |
| 473 | + | if d.purgeAdmissionClosedLocked() { | |
| 474 | + | d.mu.Unlock() | |
| 475 | + | return errPurgeAdmissionClosed | |
| 476 | + | } | |
| 473 | 477 | if d.nameLiveOrStoppedLocked(name) { | |
| 474 | 478 | d.mu.Unlock() | |
| 475 | 479 | return errSessionNameInUse | |
| @@ -542,6 +546,9 @@ func (d *Daemon) createSessionAndSwitchForAttachment(effect *attachmentEffect, n | |||
| 542 | 546 | if d.closing { | |
| 543 | 547 | return nil, errors.New("daemon is shutting down") | |
| 544 | 548 | } | |
| 549 | + | if d.purgeAdmissionClosedLocked() { | |
| 550 | + | return nil, errPurgeAdmissionClosed | |
| 551 | + | } | |
| 545 | 552 | if d.nameLiveOrStoppedLocked(name) { | |
| 546 | 553 | return nil, errSessionNameInUse | |
| 547 | 554 | } | |
| @@ -574,6 +581,10 @@ func (d *Daemon) createEphemeralSessionAndSwitch(from *session, ac *attachedClie | |||
| 574 | 581 | d.mu.Unlock() | |
| 575 | 582 | return errors.New("daemon is shutting down") | |
| 576 | 583 | } | |
| 584 | + | if d.purgeAdmissionClosedLocked() { | |
| 585 | + | d.mu.Unlock() | |
| 586 | + | return errPurgeAdmissionClosed | |
| 587 | + | } | |
| 577 | 588 | from.mu.Lock() | |
| 578 | 589 | cwd, env := from.cwd, copyEnvironment(from.env) | |
| 579 | 590 | _, attached := from.attachments[ac] | |
| @@ -620,6 +631,9 @@ func (d *Daemon) createEphemeralSessionAndSwitchForAttachment(effect *attachment | |||
| 620 | 631 | if capability.sess == nil || d.closing { | |
| 621 | 632 | return nil, errAttachmentTransition | |
| 622 | 633 | } | |
| 634 | + | if d.purgeAdmissionClosedLocked() { | |
| 635 | + | return nil, errPurgeAdmissionClosed | |
| 636 | + | } | |
| 623 | 637 | source := capability.sess | |
| 624 | 638 | source.mu.Lock() | |
| 625 | 639 | cwd, env := source.cwd, copyEnvironment(source.env) | |
| @@ -1180,8 +1194,7 @@ func (d *Daemon) closeTabLockedWithEffect(sess *session, tb *tab, repaint bool, | |||
| 1180 | 1194 | } | |
| 1181 | 1195 | if err != nil { | |
| 1182 | 1196 | d.log.Warn("closing last tab failed", "session", name, "err", err) | |
| 1183 | - | d.reportError(sess, domain.UserErr(domain.NoticeSnapshotSaturated, | |
| 1184 | - | "couldn't close tab: session state not yet saved; try again", err)) | |
| 1197 | + | d.reportError(sess, closeTabFailureNotice(err)) | |
| 1185 | 1198 | return err | |
| 1186 | 1199 | } | |
| 1187 | 1200 | d.log.Info("tab closed", "session", name, "last", true) | |
| @@ -1230,6 +1243,19 @@ func (d *Daemon) closeTabLockedWithEffect(sess *session, tb *tab, repaint bool, | |||
| 1230 | 1243 | return nil | |
| 1231 | 1244 | } | |
| 1232 | 1245 | ||
| 1246 | + | // closeTabFailureNotice classifies a failed last-tab close for the attached | |
| 1247 | + | // client. A participants-changed abort removed nothing and is safe to retry, so | |
| 1248 | + | // it reports that retryable outcome instead of the snapshot-write failure the | |
| 1249 | + | // other teardown errors carry. | |
| 1250 | + | func closeTabFailureNotice(err error) error { | |
| 1251 | + | if errors.Is(err, errSessionKillParticipantsChanged) { | |
| 1252 | + | return domain.UserErr(domain.NoticeSessionUnavailable, | |
| 1253 | + | "couldn't close tab: session changed during the close; try again", err) | |
| 1254 | + | } | |
| 1255 | + | return domain.UserErr(domain.NoticeSnapshotSaturated, | |
| 1256 | + | "couldn't close tab: session state not yet saved; try again", err) | |
| 1257 | + | } | |
| 1258 | + | ||
| 1233 | 1259 | // ptyReader drains child output into the VT screen and pokes the dirty channel | |
| 1234 | 1260 | // (non-blocking: a full channel already means a render is pending). On any read | |
| 1235 | 1261 | ||
| @@ -1310,6 +1336,15 @@ func (d *Daemon) retryStoppedPurgeContextExact(ctx context.Context, name string, | |||
| 1310 | 1336 | return nil | |
| 1311 | 1337 | } | |
| 1312 | 1338 | ||
| 1339 | + | // killSession removes a session and tears down its resources. It is | |
| 1340 | + | // idempotent: only the caller that wins the registry delete acts. Final | |
| 1341 | + | // session removal is independent of daemon shutdown: an empty registry leaves | |
| 1342 | + | // the daemon serving (closing is untouched) so a later create/restore can | |
| 1343 | + | // occupy it again, and only an explicit shutdownAll ends the daemon. | |
| 1344 | + | // | |
| 1345 | + | // Teardown ordering matters: context cancel and pty.Close run first and | |
| 1346 | + | // unconditionally — never gated behind a client send. The Detached notice is | |
| 1347 | + | // delivered asynchronously under the drain deadline. | |
| 1313 | 1348 | func (d *Daemon) killSession(sess *session, reason uint8, purge bool) error { | |
| 1314 | 1349 | return d.killSessionWithSnapshotDeadline(sess, reason, purge, nil, nil) | |
| 1315 | 1350 | } | |
| @@ -1408,6 +1443,18 @@ func (d *Daemon) snapshotSessionKillParticipants(target *session, admission *ses | |||
| 1408 | 1443 | return snapshot, true | |
| 1409 | 1444 | } | |
| 1410 | 1445 | ||
| 1446 | + | // sessionKillTargetRemoved reports whether the exact target session has already | |
| 1447 | + | // left the live registry, so a post-freeze mismatch is an already-removed no-op | |
| 1448 | + | // rather than a participant change that left work undone. | |
| 1449 | + | func (d *Daemon) sessionKillTargetRemoved(target *session) bool { | |
| 1450 | + | if target == nil { | |
| 1451 | + | return true | |
| 1452 | + | } | |
| 1453 | + | d.mu.Lock() | |
| 1454 | + | defer d.mu.Unlock() | |
| 1455 | + | return d.sessions[target.id] != target | |
| 1456 | + | } | |
| 1457 | + | ||
| 1411 | 1458 | // sessionKillParticipantsCurrent repeats lifecycle, source-token, and exact | |
| 1412 | 1459 | // target-membership validation while every discovered attachment gate is frozen. | |
| 1413 | 1460 | func (d *Daemon) sessionKillParticipantsCurrent(snapshot sessionKillParticipants) bool { | |
| @@ -1580,10 +1627,61 @@ func (s *session) finishTeardown() { | |||
| 1580 | 1627 | s.teardownMu.Unlock() | |
| 1581 | 1628 | } | |
| 1582 | 1629 | ||
| 1630 | + | // errSessionKillDeadline reports that a caller-supplied teardown deadline | |
| 1631 | + | // expired before killSessionWithSnapshotDeadlineAndCondition froze every | |
| 1632 | + | // attachment gate, acquired teardown ownership, or finished joining the | |
| 1633 | + | // session's snapshot publication. The session is left live and registered | |
| 1634 | + | // exactly as it was: a purge's snapshot quarantine is rolled back, restoring | |
| 1635 | + | // eligibility and dirty scheduling for the surviving session, so the abort is | |
| 1636 | + | // retry state rather than a lifecycle mutation. It is deliberately distinct | |
| 1637 | + | // from a completed teardown: a bounded purge or shutdown must report the skip as | |
| 1638 | + | // a typed failure or an incomplete verdict instead of misreporting success. | |
| 1639 | + | // Callers that pass no deadline never observe it. | |
| 1640 | + | var errSessionKillDeadline = errors.New("session teardown deadline exceeded before ownership") | |
| 1641 | + | ||
| 1642 | + | // errSessionKillParticipantsChanged reports that a concurrent detach or token | |
| 1643 | + | // resume changed the participant set between the pre-freeze snapshot and a | |
| 1644 | + | // post-freeze revalidation, so the kill aborted before freezing every gate or | |
| 1645 | + | // before publishing terminal ownership. The session was left untouched. It is | |
| 1646 | + | // deliberately distinct from a completed teardown so a bounded purge can retry | |
| 1647 | + | // against a fresh snapshot instead of counting the abort as a purged unit, and | |
| 1648 | + | // so a caller that does not retry still sees that nothing was removed. | |
| 1649 | + | var errSessionKillParticipantsChanged = errors.New("session kill participants changed after snapshot") | |
| 1650 | + | ||
| 1651 | + | // sessionKillRetryMessage is the user-facing, retryable outcome for a teardown | |
| 1652 | + | // a concurrent detach or token resume invalidated before it removed anything. | |
| 1653 | + | // The sentinel above is internal bookkeeping, so every user-facing path (a | |
| 1654 | + | // direct kill or a close-tab command) maps it to this message instead of | |
| 1655 | + | // leaking the sentinel text or a permanent-failure verdict. | |
| 1656 | + | const sessionKillRetryMessage = "session changed during the operation; try again" | |
| 1657 | + | ||
| 1658 | + | // teardownDeadlineElapsed reports whether the supplied deadline has already | |
| 1659 | + | // expired. A nil deadline never counts as elapsed. | |
| 1660 | + | func teardownDeadlineElapsed(deadline *snapshotShutdownDeadline) bool { | |
| 1661 | + | if deadline == nil { | |
| 1662 | + | return false | |
| 1663 | + | } | |
| 1664 | + | select { | |
| 1665 | + | case <-deadline.Done(): | |
| 1666 | + | return true | |
| 1667 | + | default: | |
| 1668 | + | return false | |
| 1669 | + | } | |
| 1670 | + | } | |
| 1671 | + | ||
| 1583 | 1672 | // killSessionWithSnapshotDeadline shares Serve's shutdown budget with its | |
| 1584 | 1673 | // terminal checkpoint and coordinator join. A timed-out repository call keeps | |
| 1585 | 1674 | // only immutable capture state; it never observes closed worker channels or | |
| 1586 | 1675 | // session-owned cache that teardown has released. | |
| 1676 | + | // | |
| 1677 | + | // A deadline that wins before this call freezes every attachment gate or | |
| 1678 | + | // acquires teardown ownership returns errSessionKillDeadline, so a bounded | |
| 1679 | + | // caller can distinguish a skip from a completed teardown. An already-removed | |
| 1680 | + | // target is a no-op; a target that is still registered while its participant | |
| 1681 | + | // set, transport incarnation, or source capability changed after the snapshot | |
| 1682 | + | // returns errSessionKillParticipantsChanged, so an aborted teardown is never | |
| 1683 | + | // mistaken for a completed one. Ordinary callers that pass nil preserve their | |
| 1684 | + | // previous behavior exactly. | |
| 1587 | 1685 | func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, purge bool, deadline *snapshotShutdownDeadline, admission *sessionKillAdmission) error { | |
| 1588 | 1686 | return d.killSessionWithSnapshotDeadlineAndCondition(sess, reason, purge, deadline, admission, false) | |
| 1589 | 1687 | } | |
| @@ -1635,10 +1733,19 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||
| 1635 | 1733 | // The acquisition helper has already rolled back exactly the gates it | |
| 1636 | 1734 | // owned. Do not enter architecture locks or publish terminal ownership | |
| 1637 | 1735 | // changes unless the complete canonical participant set is frozen. | |
| 1736 | + | if teardownDeadlineElapsed(deadline) { | |
| 1737 | + | return errSessionKillDeadline | |
| 1738 | + | } | |
| 1638 | 1739 | return nil | |
| 1639 | 1740 | } | |
| 1640 | 1741 | if !d.sessionKillParticipantsCurrent(participants) { | |
| 1641 | - | return nil | |
| 1742 | + | if d.sessionKillTargetRemoved(sess) { | |
| 1743 | + | // The exact target already left the registry, so there is nothing left | |
| 1744 | + | // to tear down: an already-removed session is a no-op, not a | |
| 1745 | + | // participant change. | |
| 1746 | + | return nil | |
| 1747 | + | } | |
| 1748 | + | return errSessionKillParticipantsChanged | |
| 1642 | 1749 | } | |
| 1643 | 1750 | ||
| 1644 | 1751 | // Pane EOF, explicit close, and daemon shutdown can all converge on the same | |
| @@ -1649,9 +1756,28 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||
| 1649 | 1756 | var acquired bool | |
| 1650 | 1757 | reason, purge, acquired = sess.beginTeardownRequest(deadline, reason, purge) | |
| 1651 | 1758 | if !acquired { | |
| 1759 | + | // beginTeardownRequest only fails to acquire when the deadline expired | |
| 1760 | + | // (without one it waits until ownership is available), so the session is | |
| 1761 | + | // still untouched and the skip must stay observable. | |
| 1762 | + | if deadline != nil { | |
| 1763 | + | return errSessionKillDeadline | |
| 1764 | + | } | |
| 1652 | 1765 | return nil | |
| 1653 | 1766 | } | |
| 1654 | 1767 | defer sess.finishTeardown() | |
| 1768 | + | // Capture eligibility before any coordinator quarantine flips it. Every | |
| 1769 | + | // post-quarantine abort that leaves this session registered restores exactly | |
| 1770 | + | // this value, so a surviving session resumes checkpoint scheduling instead of | |
| 1771 | + | // staying permanently frozen. The rollback is gated on the same teardown | |
| 1772 | + | // ownership as finishTeardown and on the exact registry entry, so a completed | |
| 1773 | + | // removal never resurrects scheduling. | |
| 1774 | + | wasEligible := sess.snapEligible.Load() | |
| 1775 | + | quarantined := false | |
| 1776 | + | defer func() { | |
| 1777 | + | if quarantined && d.sessionRegistered(sess) { | |
| 1778 | + | rollbackSnapshotQuarantine(sess, wasEligible) | |
| 1779 | + | } | |
| 1780 | + | }() | |
| 1655 | 1781 | ||
| 1656 | 1782 | d.mu.Lock() | |
| 1657 | 1783 | current := d.sessions[sess.id] | |
| @@ -1673,13 +1799,26 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||
| 1673 | 1799 | incarnation := sess.incarnation | |
| 1674 | 1800 | sess.mu.Unlock() | |
| 1675 | 1801 | // Join snapshot publication before changing live identity or deleting the | |
| 1676 | - | // incarnation directory, so an older publication cannot recreate it. | |
| 1802 | + | // incarnation directory, so an older publication cannot recreate it. A shared | |
| 1803 | + | // caller deadline bounds the join: a repository call that ignores cancellation | |
| 1804 | + | // must never hold purge admission open, so an expired budget leaves the unit | |
| 1805 | + | // live and reports the typed skip instead. | |
| 1677 | 1806 | if purge && !isEphemeral { | |
| 1678 | 1807 | if err := d.beginSnapshotPurge(name, incarnation); err != nil { | |
| 1679 | 1808 | return err | |
| 1680 | 1809 | } | |
| 1681 | 1810 | if d.snapshotRepository != nil { | |
| 1682 | - | <-quarantineSnapshotCoordinator(sess) | |
| 1811 | + | quarantineDone := quarantineSnapshotCoordinator(sess) | |
| 1812 | + | quarantined = true | |
| 1813 | + | if deadline != nil { | |
| 1814 | + | select { | |
| 1815 | + | case <-quarantineDone: | |
| 1816 | + | case <-deadline.Done(): | |
| 1817 | + | return errSessionKillDeadline | |
| 1818 | + | } | |
| 1819 | + | } else { | |
| 1820 | + | <-quarantineDone | |
| 1821 | + | } | |
| 1683 | 1822 | } | |
| 1684 | 1823 | } | |
| 1685 | 1824 | if !purge && !isEphemeral && d.persistEnabled { | |
| @@ -1719,6 +1858,7 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||
| 1719 | 1858 | } else { | |
| 1720 | 1859 | quarantineDone = quarantineSnapshotCoordinator(sess) | |
| 1721 | 1860 | } | |
| 1861 | + | quarantined = true | |
| 1722 | 1862 | joined := true | |
| 1723 | 1863 | if deadline != nil { | |
| 1724 | 1864 | select { | |
| @@ -1734,6 +1874,9 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||
| 1734 | 1874 | sess.snapshotChunkCache = nil | |
| 1735 | 1875 | sess.snapshotMu.Unlock() | |
| 1736 | 1876 | } | |
| 1877 | + | if d.afterSessionKillQuarantine != nil { | |
| 1878 | + | d.afterSessionKillQuarantine(sess) | |
| 1879 | + | } | |
| 1737 | 1880 | } | |
| 1738 | 1881 | // Revalidate the exact source capability, target lifecycle, and attachment | |
| 1739 | 1882 | // memberships under canonical architecture locks immediately before | |
| @@ -1757,7 +1900,7 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||
| 1757 | 1900 | unlockSessions() | |
| 1758 | 1901 | d.notices.routingMu.Unlock() | |
| 1759 | 1902 | d.mu.Unlock() | |
| 1760 | - | return nil | |
| 1903 | + | return errSessionKillParticipantsChanged | |
| 1761 | 1904 | } | |
| 1762 | 1905 | ||
| 1763 | 1906 | // d.mu -> routingMu -> ordered sessions -> ordered coordinators are held | |
| @@ -1798,13 +1941,6 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||
| 1798 | 1941 | stopped.purging = purge | |
| 1799 | 1942 | d.inactive[stoppedName] = stopped | |
| 1800 | 1943 | } | |
| 1801 | - | empty := len(d.sessions) == 0 | |
| 1802 | - | if empty { | |
| 1803 | - | // Shutdown is now inevitable and irreversible (doneOnce below): stop | |
| 1804 | - | // route from inserting new sessions from this instant, while we still | |
| 1805 | - | // hold the same lock route checks under. | |
| 1806 | - | d.closing = true | |
| 1807 | - | } | |
| 1808 | 1944 | d.mu.Unlock() | |
| 1809 | 1945 | d.finishParkedAttachmentRetirements(parkedRetirements) | |
| 1810 | 1946 | frozen.unfreeze() | |
| @@ -1844,27 +1980,23 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||
| 1844 | 1980 | } | |
| 1845 | 1981 | // The coordinator and all panes are now stopped, so no producer can publish | |
| 1846 | 1982 | // another generation after this destructive source deletion. Keep the hidden | |
| 1847 | - | // stopped record when deletion fails so a repeated live kill can retry. | |
| 1983 | + | // stopped record when deletion fails or is still pending so a repeated live | |
| 1984 | + | // kill can retry. The purge-owned exact delete is bounded: a repository | |
| 1985 | + | // delete that ignores cancellation is detached behind a name/incarnation/ | |
| 1986 | + | // createdAt fence rather than holding purgeAllMu, admission, or control | |
| 1987 | + | // responses. | |
| 1848 | 1988 | if !ephemeral && purge { | |
| 1849 | - | err := d.finishSnapshotPurge(d.serveCtx, stoppedName, incarnation, stoppedRecord.CreatedAt) | |
| 1850 | - | if err != nil { | |
| 1989 | + | if err := d.finishLiveDurablePurge(deadline, stoppedRecord); err != nil { | |
| 1851 | 1990 | purgeErr = errors.Join(purgeErr, err) | |
| 1852 | 1991 | d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName) | |
| 1853 | - | } else { | |
| 1854 | - | expected := inactiveSessionFromRecord(stoppedRecord, protocol.SessionDown, nil) | |
| 1855 | - | d.mu.Lock() | |
| 1856 | - | if stopped, ok := d.inactive[stoppedName]; ok && stopped.purging && stopped.sameLifecycle(expected) { | |
| 1857 | - | delete(d.inactive, stoppedName) | |
| 1858 | - | } | |
| 1859 | - | d.mu.Unlock() | |
| 1860 | 1992 | } | |
| 1861 | 1993 | } | |
| 1862 | - | if purge && purgeErr == nil { | |
| 1994 | + | if purge && purgeErr == nil && isEphemeral { | |
| 1995 | + | // An ephemeral purge owns no durable record, but the remaining sessions' | |
| 1996 | + | // route histories may still reference it. | |
| 1863 | 1997 | d.reconcileAllRouteHistories() | |
| 1864 | 1998 | } | |
| 1865 | - | if empty { | |
| 1866 | - | d.doneOnce.Do(func() { close(d.done) }) | |
| 1867 | - | } else if ringing { | |
| 1999 | + | if ringing { | |
| 1868 | 2000 | d.repaintAllAttachedClients() | |
| 1869 | 2001 | } | |
| 1870 | 2002 | ||
| @@ -470,6 +470,10 @@ func (d *Daemon) createSessionAndSwitch(from *session, ac *attachedClient, name | |||||
| 470 | d.mu.Unlock() | 470 | d.mu.Unlock() | ||
| 471 | return errors.New("daemon is shutting down") | 471 | return errors.New("daemon is shutting down") | ||
| 472 | } | 472 | } | ||
| 473 | + | if d.purgeAdmissionClosedLocked() { | |||
| 474 | + | d.mu.Unlock() | |||
| 475 | + | return errPurgeAdmissionClosed | |||
| 476 | + | } | |||
| 473 | if d.nameLiveOrStoppedLocked(name) { | 477 | if d.nameLiveOrStoppedLocked(name) { | ||
| 474 | d.mu.Unlock() | 478 | d.mu.Unlock() | ||
| 475 | return errSessionNameInUse | 479 | return errSessionNameInUse | ||
| @@ -542,6 +546,9 @@ func (d *Daemon) createSessionAndSwitchForAttachment(effect *attachmentEffect, n | |||||
| 542 | if d.closing { | 546 | if d.closing { | ||
| 543 | return nil, errors.New("daemon is shutting down") | 547 | return nil, errors.New("daemon is shutting down") | ||
| 544 | } | 548 | } | ||
| 549 | + | if d.purgeAdmissionClosedLocked() { | |||
| 550 | + | return nil, errPurgeAdmissionClosed | |||
| 551 | + | } | |||
| 545 | if d.nameLiveOrStoppedLocked(name) { | 552 | if d.nameLiveOrStoppedLocked(name) { | ||
| 546 | return nil, errSessionNameInUse | 553 | return nil, errSessionNameInUse | ||
| 547 | } | 554 | } | ||
| @@ -574,6 +581,10 @@ func (d *Daemon) createEphemeralSessionAndSwitch(from *session, ac *attachedClie | |||||
| 574 | d.mu.Unlock() | 581 | d.mu.Unlock() | ||
| 575 | return errors.New("daemon is shutting down") | 582 | return errors.New("daemon is shutting down") | ||
| 576 | } | 583 | } | ||
| 584 | + | if d.purgeAdmissionClosedLocked() { | |||
| 585 | + | d.mu.Unlock() | |||
| 586 | + | return errPurgeAdmissionClosed | |||
| 587 | + | } | |||
| 577 | from.mu.Lock() | 588 | from.mu.Lock() | ||
| 578 | cwd, env := from.cwd, copyEnvironment(from.env) | 589 | cwd, env := from.cwd, copyEnvironment(from.env) | ||
| 579 | _, attached := from.attachments[ac] | 590 | _, attached := from.attachments[ac] | ||
| @@ -620,6 +631,9 @@ func (d *Daemon) createEphemeralSessionAndSwitchForAttachment(effect *attachment | |||||
| 620 | if capability.sess == nil || d.closing { | 631 | if capability.sess == nil || d.closing { | ||
| 621 | return nil, errAttachmentTransition | 632 | return nil, errAttachmentTransition | ||
| 622 | } | 633 | } | ||
| 634 | + | if d.purgeAdmissionClosedLocked() { | |||
| 635 | + | return nil, errPurgeAdmissionClosed | |||
| 636 | + | } | |||
| 623 | source := capability.sess | 637 | source := capability.sess | ||
| 624 | source.mu.Lock() | 638 | source.mu.Lock() | ||
| 625 | cwd, env := source.cwd, copyEnvironment(source.env) | 639 | cwd, env := source.cwd, copyEnvironment(source.env) | ||
| @@ -1180,8 +1194,7 @@ func (d *Daemon) closeTabLockedWithEffect(sess *session, tb *tab, repaint bool, | |||||
| 1180 | } | 1194 | } | ||
| 1181 | if err != nil { | 1195 | if err != nil { | ||
| 1182 | d.log.Warn("closing last tab failed", "session", name, "err", err) | 1196 | d.log.Warn("closing last tab failed", "session", name, "err", err) | ||
| 1183 | - | d.reportError(sess, domain.UserErr(domain.NoticeSnapshotSaturated, | 1197 | + | d.reportError(sess, closeTabFailureNotice(err)) |
| 1184 | - | "couldn't close tab: session state not yet saved; try again", err)) | |||
| 1185 | return err | 1198 | return err | ||
| 1186 | } | 1199 | } | ||
| 1187 | d.log.Info("tab closed", "session", name, "last", true) | 1200 | d.log.Info("tab closed", "session", name, "last", true) | ||
| @@ -1230,6 +1243,19 @@ func (d *Daemon) closeTabLockedWithEffect(sess *session, tb *tab, repaint bool, | |||||
| 1230 | return nil | 1243 | return nil | ||
| 1231 | } | 1244 | } | ||
| 1232 | 1245 | ||||
| 1246 | + | // closeTabFailureNotice classifies a failed last-tab close for the attached | |||
| 1247 | + | // client. A participants-changed abort removed nothing and is safe to retry, so | |||
| 1248 | + | // it reports that retryable outcome instead of the snapshot-write failure the | |||
| 1249 | + | // other teardown errors carry. | |||
| 1250 | + | func closeTabFailureNotice(err error) error { | |||
| 1251 | + | if errors.Is(err, errSessionKillParticipantsChanged) { | |||
| 1252 | + | return domain.UserErr(domain.NoticeSessionUnavailable, | |||
| 1253 | + | "couldn't close tab: session changed during the close; try again", err) | |||
| 1254 | + | } | |||
| 1255 | + | return domain.UserErr(domain.NoticeSnapshotSaturated, | |||
| 1256 | + | "couldn't close tab: session state not yet saved; try again", err) | |||
| 1257 | + | } | |||
| 1258 | + | ||||
| 1233 | // ptyReader drains child output into the VT screen and pokes the dirty channel | 1259 | // ptyReader drains child output into the VT screen and pokes the dirty channel | ||
| 1234 | // (non-blocking: a full channel already means a render is pending). On any read | 1260 | // (non-blocking: a full channel already means a render is pending). On any read | ||
| 1235 | 1261 | ||||
| @@ -1310,6 +1336,15 @@ func (d *Daemon) retryStoppedPurgeContextExact(ctx context.Context, name string, | |||||
| 1310 | return nil | 1336 | return nil | ||
| 1311 | } | 1337 | } | ||
| 1312 | 1338 | ||||
| 1339 | + | // killSession removes a session and tears down its resources. It is | |||
| 1340 | + | // idempotent: only the caller that wins the registry delete acts. Final | |||
| 1341 | + | // session removal is independent of daemon shutdown: an empty registry leaves | |||
| 1342 | + | // the daemon serving (closing is untouched) so a later create/restore can | |||
| 1343 | + | // occupy it again, and only an explicit shutdownAll ends the daemon. | |||
| 1344 | + | // | |||
| 1345 | + | // Teardown ordering matters: context cancel and pty.Close run first and | |||
| 1346 | + | // unconditionally — never gated behind a client send. The Detached notice is | |||
| 1347 | + | // delivered asynchronously under the drain deadline. | |||
| 1313 | func (d *Daemon) killSession(sess *session, reason uint8, purge bool) error { | 1348 | func (d *Daemon) killSession(sess *session, reason uint8, purge bool) error { | ||
| 1314 | return d.killSessionWithSnapshotDeadline(sess, reason, purge, nil, nil) | 1349 | return d.killSessionWithSnapshotDeadline(sess, reason, purge, nil, nil) | ||
| 1315 | } | 1350 | } | ||
| @@ -1408,6 +1443,18 @@ func (d *Daemon) snapshotSessionKillParticipants(target *session, admission *ses | |||||
| 1408 | return snapshot, true | 1443 | return snapshot, true | ||
| 1409 | } | 1444 | } | ||
| 1410 | 1445 | ||||
| 1446 | + | // sessionKillTargetRemoved reports whether the exact target session has already | |||
| 1447 | + | // left the live registry, so a post-freeze mismatch is an already-removed no-op | |||
| 1448 | + | // rather than a participant change that left work undone. | |||
| 1449 | + | func (d *Daemon) sessionKillTargetRemoved(target *session) bool { | |||
| 1450 | + | if target == nil { | |||
| 1451 | + | return true | |||
| 1452 | + | } | |||
| 1453 | + | d.mu.Lock() | |||
| 1454 | + | defer d.mu.Unlock() | |||
| 1455 | + | return d.sessions[target.id] != target | |||
| 1456 | + | } | |||
| 1457 | + | ||||
| 1411 | // sessionKillParticipantsCurrent repeats lifecycle, source-token, and exact | 1458 | // sessionKillParticipantsCurrent repeats lifecycle, source-token, and exact | ||
| 1412 | // target-membership validation while every discovered attachment gate is frozen. | 1459 | // target-membership validation while every discovered attachment gate is frozen. | ||
| 1413 | func (d *Daemon) sessionKillParticipantsCurrent(snapshot sessionKillParticipants) bool { | 1460 | func (d *Daemon) sessionKillParticipantsCurrent(snapshot sessionKillParticipants) bool { | ||
| @@ -1580,10 +1627,61 @@ func (s *session) finishTeardown() { | |||||
| 1580 | s.teardownMu.Unlock() | 1627 | s.teardownMu.Unlock() | ||
| 1581 | } | 1628 | } | ||
| 1582 | 1629 | ||||
| 1630 | + | // errSessionKillDeadline reports that a caller-supplied teardown deadline | |||
| 1631 | + | // expired before killSessionWithSnapshotDeadlineAndCondition froze every | |||
| 1632 | + | // attachment gate, acquired teardown ownership, or finished joining the | |||
| 1633 | + | // session's snapshot publication. The session is left live and registered | |||
| 1634 | + | // exactly as it was: a purge's snapshot quarantine is rolled back, restoring | |||
| 1635 | + | // eligibility and dirty scheduling for the surviving session, so the abort is | |||
| 1636 | + | // retry state rather than a lifecycle mutation. It is deliberately distinct | |||
| 1637 | + | // from a completed teardown: a bounded purge or shutdown must report the skip as | |||
| 1638 | + | // a typed failure or an incomplete verdict instead of misreporting success. | |||
| 1639 | + | // Callers that pass no deadline never observe it. | |||
| 1640 | + | var errSessionKillDeadline = errors.New("session teardown deadline exceeded before ownership") | |||
| 1641 | + | ||||
| 1642 | + | // errSessionKillParticipantsChanged reports that a concurrent detach or token | |||
| 1643 | + | // resume changed the participant set between the pre-freeze snapshot and a | |||
| 1644 | + | // post-freeze revalidation, so the kill aborted before freezing every gate or | |||
| 1645 | + | // before publishing terminal ownership. The session was left untouched. It is | |||
| 1646 | + | // deliberately distinct from a completed teardown so a bounded purge can retry | |||
| 1647 | + | // against a fresh snapshot instead of counting the abort as a purged unit, and | |||
| 1648 | + | // so a caller that does not retry still sees that nothing was removed. | |||
| 1649 | + | var errSessionKillParticipantsChanged = errors.New("session kill participants changed after snapshot") | |||
| 1650 | + | ||||
| 1651 | + | // sessionKillRetryMessage is the user-facing, retryable outcome for a teardown | |||
| 1652 | + | // a concurrent detach or token resume invalidated before it removed anything. | |||
| 1653 | + | // The sentinel above is internal bookkeeping, so every user-facing path (a | |||
| 1654 | + | // direct kill or a close-tab command) maps it to this message instead of | |||
| 1655 | + | // leaking the sentinel text or a permanent-failure verdict. | |||
| 1656 | + | const sessionKillRetryMessage = "session changed during the operation; try again" | |||
| 1657 | + | ||||
| 1658 | + | // teardownDeadlineElapsed reports whether the supplied deadline has already | |||
| 1659 | + | // expired. A nil deadline never counts as elapsed. | |||
| 1660 | + | func teardownDeadlineElapsed(deadline *snapshotShutdownDeadline) bool { | |||
| 1661 | + | if deadline == nil { | |||
| 1662 | + | return false | |||
| 1663 | + | } | |||
| 1664 | + | select { | |||
| 1665 | + | case <-deadline.Done(): | |||
| 1666 | + | return true | |||
| 1667 | + | default: | |||
| 1668 | + | return false | |||
| 1669 | + | } | |||
| 1670 | + | } | |||
| 1671 | + | ||||
| 1583 | // killSessionWithSnapshotDeadline shares Serve's shutdown budget with its | 1672 | // killSessionWithSnapshotDeadline shares Serve's shutdown budget with its | ||
| 1584 | // terminal checkpoint and coordinator join. A timed-out repository call keeps | 1673 | // terminal checkpoint and coordinator join. A timed-out repository call keeps | ||
| 1585 | // only immutable capture state; it never observes closed worker channels or | 1674 | // only immutable capture state; it never observes closed worker channels or | ||
| 1586 | // session-owned cache that teardown has released. | 1675 | // session-owned cache that teardown has released. | ||
| 1676 | + | // | |||
| 1677 | + | // A deadline that wins before this call freezes every attachment gate or | |||
| 1678 | + | // acquires teardown ownership returns errSessionKillDeadline, so a bounded | |||
| 1679 | + | // caller can distinguish a skip from a completed teardown. An already-removed | |||
| 1680 | + | // target is a no-op; a target that is still registered while its participant | |||
| 1681 | + | // set, transport incarnation, or source capability changed after the snapshot | |||
| 1682 | + | // returns errSessionKillParticipantsChanged, so an aborted teardown is never | |||
| 1683 | + | // mistaken for a completed one. Ordinary callers that pass nil preserve their | |||
| 1684 | + | // previous behavior exactly. | |||
| 1587 | func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, purge bool, deadline *snapshotShutdownDeadline, admission *sessionKillAdmission) error { | 1685 | func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, purge bool, deadline *snapshotShutdownDeadline, admission *sessionKillAdmission) error { | ||
| 1588 | return d.killSessionWithSnapshotDeadlineAndCondition(sess, reason, purge, deadline, admission, false) | 1686 | return d.killSessionWithSnapshotDeadlineAndCondition(sess, reason, purge, deadline, admission, false) | ||
| 1589 | } | 1687 | } | ||
| @@ -1635,10 +1733,19 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||||
| 1635 | // The acquisition helper has already rolled back exactly the gates it | 1733 | // The acquisition helper has already rolled back exactly the gates it | ||
| 1636 | // owned. Do not enter architecture locks or publish terminal ownership | 1734 | // owned. Do not enter architecture locks or publish terminal ownership | ||
| 1637 | // changes unless the complete canonical participant set is frozen. | 1735 | // changes unless the complete canonical participant set is frozen. | ||
| 1736 | + | if teardownDeadlineElapsed(deadline) { | |||
| 1737 | + | return errSessionKillDeadline | |||
| 1738 | + | } | |||
| 1638 | return nil | 1739 | return nil | ||
| 1639 | } | 1740 | } | ||
| 1640 | if !d.sessionKillParticipantsCurrent(participants) { | 1741 | if !d.sessionKillParticipantsCurrent(participants) { | ||
| 1641 | - | return nil | 1742 | + | if d.sessionKillTargetRemoved(sess) { |
| 1743 | + | // The exact target already left the registry, so there is nothing left | |||
| 1744 | + | // to tear down: an already-removed session is a no-op, not a | |||
| 1745 | + | // participant change. | |||
| 1746 | + | return nil | |||
| 1747 | + | } | |||
| 1748 | + | return errSessionKillParticipantsChanged | |||
| 1642 | } | 1749 | } | ||
| 1643 | 1750 | ||||
| 1644 | // Pane EOF, explicit close, and daemon shutdown can all converge on the same | 1751 | // Pane EOF, explicit close, and daemon shutdown can all converge on the same | ||
| @@ -1649,9 +1756,28 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||||
| 1649 | var acquired bool | 1756 | var acquired bool | ||
| 1650 | reason, purge, acquired = sess.beginTeardownRequest(deadline, reason, purge) | 1757 | reason, purge, acquired = sess.beginTeardownRequest(deadline, reason, purge) | ||
| 1651 | if !acquired { | 1758 | if !acquired { | ||
| 1759 | + | // beginTeardownRequest only fails to acquire when the deadline expired | |||
| 1760 | + | // (without one it waits until ownership is available), so the session is | |||
| 1761 | + | // still untouched and the skip must stay observable. | |||
| 1762 | + | if deadline != nil { | |||
| 1763 | + | return errSessionKillDeadline | |||
| 1764 | + | } | |||
| 1652 | return nil | 1765 | return nil | ||
| 1653 | } | 1766 | } | ||
| 1654 | defer sess.finishTeardown() | 1767 | defer sess.finishTeardown() | ||
| 1768 | + | // Capture eligibility before any coordinator quarantine flips it. Every | |||
| 1769 | + | // post-quarantine abort that leaves this session registered restores exactly | |||
| 1770 | + | // this value, so a surviving session resumes checkpoint scheduling instead of | |||
| 1771 | + | // staying permanently frozen. The rollback is gated on the same teardown | |||
| 1772 | + | // ownership as finishTeardown and on the exact registry entry, so a completed | |||
| 1773 | + | // removal never resurrects scheduling. | |||
| 1774 | + | wasEligible := sess.snapEligible.Load() | |||
| 1775 | + | quarantined := false | |||
| 1776 | + | defer func() { | |||
| 1777 | + | if quarantined && d.sessionRegistered(sess) { | |||
| 1778 | + | rollbackSnapshotQuarantine(sess, wasEligible) | |||
| 1779 | + | } | |||
| 1780 | + | }() | |||
| 1655 | 1781 | ||||
| 1656 | d.mu.Lock() | 1782 | d.mu.Lock() | ||
| 1657 | current := d.sessions[sess.id] | 1783 | current := d.sessions[sess.id] | ||
| @@ -1673,13 +1799,26 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||||
| 1673 | incarnation := sess.incarnation | 1799 | incarnation := sess.incarnation | ||
| 1674 | sess.mu.Unlock() | 1800 | sess.mu.Unlock() | ||
| 1675 | // Join snapshot publication before changing live identity or deleting the | 1801 | // Join snapshot publication before changing live identity or deleting the | ||
| 1676 | - | // incarnation directory, so an older publication cannot recreate it. | 1802 | + | // incarnation directory, so an older publication cannot recreate it. A shared |
| 1803 | + | // caller deadline bounds the join: a repository call that ignores cancellation | |||
| 1804 | + | // must never hold purge admission open, so an expired budget leaves the unit | |||
| 1805 | + | // live and reports the typed skip instead. | |||
| 1677 | if purge && !isEphemeral { | 1806 | if purge && !isEphemeral { | ||
| 1678 | if err := d.beginSnapshotPurge(name, incarnation); err != nil { | 1807 | if err := d.beginSnapshotPurge(name, incarnation); err != nil { | ||
| 1679 | return err | 1808 | return err | ||
| 1680 | } | 1809 | } | ||
| 1681 | if d.snapshotRepository != nil { | 1810 | if d.snapshotRepository != nil { | ||
| 1682 | - | <-quarantineSnapshotCoordinator(sess) | 1811 | + | quarantineDone := quarantineSnapshotCoordinator(sess) |
| 1812 | + | quarantined = true | |||
| 1813 | + | if deadline != nil { | |||
| 1814 | + | select { | |||
| 1815 | + | case <-quarantineDone: | |||
| 1816 | + | case <-deadline.Done(): | |||
| 1817 | + | return errSessionKillDeadline | |||
| 1818 | + | } | |||
| 1819 | + | } else { | |||
| 1820 | + | <-quarantineDone | |||
| 1821 | + | } | |||
| 1683 | } | 1822 | } | ||
| 1684 | } | 1823 | } | ||
| 1685 | if !purge && !isEphemeral && d.persistEnabled { | 1824 | if !purge && !isEphemeral && d.persistEnabled { | ||
| @@ -1719,6 +1858,7 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||||
| 1719 | } else { | 1858 | } else { | ||
| 1720 | quarantineDone = quarantineSnapshotCoordinator(sess) | 1859 | quarantineDone = quarantineSnapshotCoordinator(sess) | ||
| 1721 | } | 1860 | } | ||
| 1861 | + | quarantined = true | |||
| 1722 | joined := true | 1862 | joined := true | ||
| 1723 | if deadline != nil { | 1863 | if deadline != nil { | ||
| 1724 | select { | 1864 | select { | ||
| @@ -1734,6 +1874,9 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||||
| 1734 | sess.snapshotChunkCache = nil | 1874 | sess.snapshotChunkCache = nil | ||
| 1735 | sess.snapshotMu.Unlock() | 1875 | sess.snapshotMu.Unlock() | ||
| 1736 | } | 1876 | } | ||
| 1877 | + | if d.afterSessionKillQuarantine != nil { | |||
| 1878 | + | d.afterSessionKillQuarantine(sess) | |||
| 1879 | + | } | |||
| 1737 | } | 1880 | } | ||
| 1738 | // Revalidate the exact source capability, target lifecycle, and attachment | 1881 | // Revalidate the exact source capability, target lifecycle, and attachment | ||
| 1739 | // memberships under canonical architecture locks immediately before | 1882 | // memberships under canonical architecture locks immediately before | ||
| @@ -1757,7 +1900,7 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||||
| 1757 | unlockSessions() | 1900 | unlockSessions() | ||
| 1758 | d.notices.routingMu.Unlock() | 1901 | d.notices.routingMu.Unlock() | ||
| 1759 | d.mu.Unlock() | 1902 | d.mu.Unlock() | ||
| 1760 | - | return nil | 1903 | + | return errSessionKillParticipantsChanged |
| 1761 | } | 1904 | } | ||
| 1762 | 1905 | ||||
| 1763 | // d.mu -> routingMu -> ordered sessions -> ordered coordinators are held | 1906 | // d.mu -> routingMu -> ordered sessions -> ordered coordinators are held | ||
| @@ -1798,13 +1941,6 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||||
| 1798 | stopped.purging = purge | 1941 | stopped.purging = purge | ||
| 1799 | d.inactive[stoppedName] = stopped | 1942 | d.inactive[stoppedName] = stopped | ||
| 1800 | } | 1943 | } | ||
| 1801 | - | empty := len(d.sessions) == 0 | |||
| 1802 | - | if empty { | |||
| 1803 | - | // Shutdown is now inevitable and irreversible (doneOnce below): stop | |||
| 1804 | - | // route from inserting new sessions from this instant, while we still | |||
| 1805 | - | // hold the same lock route checks under. | |||
| 1806 | - | d.closing = true | |||
| 1807 | - | } | |||
| 1808 | d.mu.Unlock() | 1944 | d.mu.Unlock() | ||
| 1809 | d.finishParkedAttachmentRetirements(parkedRetirements) | 1945 | d.finishParkedAttachmentRetirements(parkedRetirements) | ||
| 1810 | frozen.unfreeze() | 1946 | frozen.unfreeze() | ||
| @@ -1844,27 +1980,23 @@ func (d *Daemon) killSessionWithSnapshotDeadlineAndCondition(sess *session, reas | |||||
| 1844 | } | 1980 | } | ||
| 1845 | // The coordinator and all panes are now stopped, so no producer can publish | 1981 | // The coordinator and all panes are now stopped, so no producer can publish | ||
| 1846 | // another generation after this destructive source deletion. Keep the hidden | 1982 | // another generation after this destructive source deletion. Keep the hidden | ||
| 1847 | - | // stopped record when deletion fails so a repeated live kill can retry. | 1983 | + | // stopped record when deletion fails or is still pending so a repeated live |
| 1984 | + | // kill can retry. The purge-owned exact delete is bounded: a repository | |||
| 1985 | + | // delete that ignores cancellation is detached behind a name/incarnation/ | |||
| 1986 | + | // createdAt fence rather than holding purgeAllMu, admission, or control | |||
| 1987 | + | // responses. | |||
| 1848 | if !ephemeral && purge { | 1988 | if !ephemeral && purge { | ||
| 1849 | - | err := d.finishSnapshotPurge(d.serveCtx, stoppedName, incarnation, stoppedRecord.CreatedAt) | 1989 | + | if err := d.finishLiveDurablePurge(deadline, stoppedRecord); err != nil { |
| 1850 | - | if err != nil { | |||
| 1851 | purgeErr = errors.Join(purgeErr, err) | 1990 | purgeErr = errors.Join(purgeErr, err) | ||
| 1852 | d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName) | 1991 | d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName) | ||
| 1853 | - | } else { | |||
| 1854 | - | expected := inactiveSessionFromRecord(stoppedRecord, protocol.SessionDown, nil) | |||
| 1855 | - | d.mu.Lock() | |||
| 1856 | - | if stopped, ok := d.inactive[stoppedName]; ok && stopped.purging && stopped.sameLifecycle(expected) { | |||
| 1857 | - | delete(d.inactive, stoppedName) | |||
| 1858 | - | } | |||
| 1859 | - | d.mu.Unlock() | |||
| 1860 | } | 1992 | } | ||
| 1861 | } | 1993 | } | ||
| 1862 | - | if purge && purgeErr == nil { | 1994 | + | if purge && purgeErr == nil && isEphemeral { |
| 1995 | + | // An ephemeral purge owns no durable record, but the remaining sessions' | |||
| 1996 | + | // route histories may still reference it. | |||
| 1863 | d.reconcileAllRouteHistories() | 1997 | d.reconcileAllRouteHistories() | ||
| 1864 | } | 1998 | } | ||
| 1865 | - | if empty { | 1999 | + | if ringing { |
| 1866 | - | d.doneOnce.Do(func() { close(d.done) }) | |||
| 1867 | - | } else if ringing { | |||
| 1868 | d.repaintAllAttachedClients() | 2000 | d.repaintAllAttachedClients() | ||
| 1869 | } | 2001 | } | ||
| 1870 | 2002 | ||||
M internal/usecase/daemon/session_test.go +208 -56
| @@ -136,16 +136,51 @@ func TestHandshakeNoSuchSession(t *testing.T) { | |||
| 136 | 136 | require.Equal(t, protocol.ErrNoSuchSession, em.Code) | |
| 137 | 137 | } | |
| 138 | 138 | ||
| 139 | - | func TestKillAllEmptyDaemonSignalsShutdown(t *testing.T) { | |
| 140 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 139 | + | // TestKillAllEmptyDaemonKeepsServing proves KillAll no longer ends the daemon: | |
| 140 | + | // purging an empty registry leaves closing, move admission, the daemon-wide | |
| 141 | + | // pane process context, and d.done untouched, and the same daemon then admits a | |
| 142 | + | // fresh session. | |
| 143 | + | func TestKillAllEmptyDaemonKeepsServing(t *testing.T) { | |
| 144 | + | pty, release := newBlockingPTY(t) | |
| 145 | + | defer release() | |
| 146 | + | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | |
| 141 | 147 | tr, _, _ := newConn(t, mustClientEnvelope(protocol.Kill{Scope: protocol.KillAll})) | |
| 142 | 148 | d.handleConn(tr) | |
| 143 | 149 | ||
| 144 | 150 | select { | |
| 145 | 151 | case <-d.done: | |
| 146 | - | case <-time.After(time.Second): | |
| 147 | - | t.Fatal("kill all on empty daemon did not signal shutdown") | |
| 152 | + | t.Fatal("kill all on empty daemon must not signal shutdown") | |
| 153 | + | default: | |
| 148 | 154 | } | |
| 155 | + | d.mu.Lock() | |
| 156 | + | closing := d.closing | |
| 157 | + | d.mu.Unlock() | |
| 158 | + | require.False(t, closing, "kill all must not mark the daemon closing") | |
| 159 | + | d.moveLifecycleMu.Lock() | |
| 160 | + | moveClosing := d.moveLifecycleClosing | |
| 161 | + | d.moveLifecycleMu.Unlock() | |
| 162 | + | require.False(t, moveClosing, "kill all must not close move admission") | |
| 163 | + | select { | |
| 164 | + | case <-d.paneProcessCtx.Done(): | |
| 165 | + | t.Fatal("kill all must not cancel the daemon-wide pane process context") | |
| 166 | + | default: | |
| 167 | + | } | |
| 168 | + | ||
| 169 | + | // The purge reopened admission: the same daemon creates a fresh session. | |
| 170 | + | tr2, sends2, releaseConn2 := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |
| 171 | + | var hg sync.WaitGroup | |
| 172 | + | hg.Go(func() { d.handleConn(tr2) }) | |
| 173 | + | welcome := decodeServerMessage(t, awaitFrame(t, sends2, "Welcome")).(protocol.Welcome) | |
| 174 | + | require.Equal(t, "0", welcome.SessionName) | |
| 175 | + | awaitFrame(t, sends2, "Output") | |
| 176 | + | require.Equal(t, 1, sessionCount(d)) | |
| 177 | + | ||
| 178 | + | releaseConn2() | |
| 179 | + | _ = d.killSession(firstSession(d), protocol.ReasonSessionKilled, true) | |
| 180 | + | release() | |
| 181 | + | hg.Wait() | |
| 182 | + | d.sessWg.Wait() | |
| 183 | + | d.waitNotifies() | |
| 149 | 184 | } | |
| 150 | 185 | ||
| 151 | 186 | func TestBeginTeardownRequestResolvesShutdownPreservation(t *testing.T) { | |
| @@ -382,14 +417,21 @@ func TestDetachKeepsNamed(t *testing.T) { | |||
| 382 | 417 | ||
| 383 | 418 | // --- render scheduler debounce ---------------------------------------------- | |
| 384 | 419 | ||
| 385 | - | func TestReaderEOFRemovesSessionAndSignalsShutdown(t *testing.T) { | |
| 386 | - | p := portsmocks.NewMockPTY(t) | |
| 420 | + | // TestReaderEOFRemovesSessionAndDaemonSurvives proves final session removal is | |
| 421 | + | // independent of daemon shutdown: the last session's EOF empties the registry | |
| 422 | + | // without signalling shutdown or marking the daemon closing, and the same | |
| 423 | + | // daemon can be occupied again by a fresh session. | |
| 424 | + | func TestReaderEOFRemovesSessionAndDaemonSurvives(t *testing.T) { | |
| 425 | + | eofPTY := portsmocks.NewMockPTY(t) | |
| 387 | 426 | // Read returns EOF immediately (child already gone). | |
| 388 | - | p.EXPECT().Read(mock.Anything).Return(0, io.EOF).Maybe() | |
| 389 | - | p.EXPECT().Close().Return(nil).Maybe() | |
| 390 | - | p.EXPECT().Pid().Return(1).Maybe() | |
| 427 | + | eofPTY.EXPECT().Read(mock.Anything).Return(0, io.EOF).Maybe() | |
| 428 | + | eofPTY.EXPECT().Close().Return(nil).Maybe() | |
| 429 | + | eofPTY.EXPECT().Pid().Return(1).Maybe() | |
| 430 | + | ||
| 431 | + | secondPTY, releaseSecond := newBlockingPTY(t) | |
| 432 | + | defer releaseSecond() | |
| 433 | + | d := newTestDaemon(t, newFactorySeq(t, eofPTY, secondPTY), stubClock{}) | |
| 391 | 434 | ||
| 392 | - | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 393 | 435 | tr, sends, _ := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |
| 394 | 436 | ||
| 395 | 437 | var hg sync.WaitGroup | |
| @@ -400,27 +442,140 @@ func TestReaderEOFRemovesSessionAndSignalsShutdown(t *testing.T) { | |||
| 400 | 442 | dm := decodeServerMessage(t, det).(protocol.Detached) | |
| 401 | 443 | require.Equal(t, protocol.ReasonSessionKilled, dm.Reason) | |
| 402 | 444 | ||
| 445 | + | // The registry is empty, yet the daemon has not begun shutdown and has not | |
| 446 | + | // signalled Serve. The Detached notice is scheduled only after registry | |
| 447 | + | // removal has published, so these assertions are deterministic. | |
| 403 | 448 | select { | |
| 404 | 449 | case <-d.done: | |
| 405 | - | case <-time.After(2 * time.Second): | |
| 406 | - | t.Fatal("registry-empty shutdown was not signalled") | |
| 450 | + | t.Fatal("final session removal must not signal daemon shutdown") | |
| 451 | + | default: | |
| 407 | 452 | } | |
| 453 | + | d.mu.Lock() | |
| 454 | + | closing := d.closing | |
| 455 | + | d.mu.Unlock() | |
| 456 | + | require.False(t, closing, "empty registry must not mark the daemon closing") | |
| 408 | 457 | require.Equal(t, 0, sessionCount(d)) | |
| 409 | 458 | ||
| 459 | + | // The empty registry can be occupied again by a brand-new session. | |
| 460 | + | tr2, sends2, releaseConn2 := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |
| 461 | + | var hg2 sync.WaitGroup | |
| 462 | + | hg2.Go(func() { d.handleConn(tr2) }) | |
| 463 | + | welcome := decodeServerMessage(t, awaitFrame(t, sends2, "Welcome")).(protocol.Welcome) | |
| 464 | + | require.Equal(t, "0", welcome.SessionName) | |
| 465 | + | awaitFrame(t, sends2, "Output") | |
| 466 | + | require.Equal(t, 1, sessionCount(d)) | |
| 467 | + | ||
| 468 | + | _ = d.killSession(firstSession(d), protocol.ReasonSessionKilled, true) | |
| 469 | + | releaseConn2() | |
| 410 | 470 | hg.Wait() | |
| 471 | + | hg2.Wait() | |
| 472 | + | d.sessWg.Wait() | |
| 473 | + | d.waitNotifies() | |
| 474 | + | } | |
| 475 | + | ||
| 476 | + | // TestConcurrentFinalRemovalAndCreationLinearize proves final-session removal | |
| 477 | + | // no longer opens a shutdown window: a create racing the removal is always | |
| 478 | + | // admitted (never rejected with ErrServerShutdown) and the two registry | |
| 479 | + | // transitions linearize under d.mu, leaving exactly the newly created session. | |
| 480 | + | func TestConcurrentFinalRemovalAndCreationLinearize(t *testing.T) { | |
| 481 | + | for attempt := 0; attempt < 32; attempt++ { | |
| 482 | + | firstPTY, releaseFirst := newBlockingPTY(t) | |
| 483 | + | secondPTY, releaseSecond := newBlockingPTY(t) | |
| 484 | + | d := newTestDaemon(t, newFactorySeq(t, firstPTY, secondPTY), stubClock{}) | |
| 485 | + | ||
| 486 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, nil) | |
| 487 | + | require.NoError(t, err) | |
| 488 | + | ||
| 489 | + | start := make(chan struct{}) | |
| 490 | + | var wg sync.WaitGroup | |
| 491 | + | wg.Add(2) | |
| 492 | + | var createErr error | |
| 493 | + | go func() { | |
| 494 | + | defer wg.Done() | |
| 495 | + | <-start | |
| 496 | + | _, _, createErr = d.route(helloResumeCapable(protocol.IntentEphemeral, "", 0), &closeTrackingTransport{}) | |
| 497 | + | }() | |
| 498 | + | go func() { | |
| 499 | + | defer wg.Done() | |
| 500 | + | <-start | |
| 501 | + | _ = d.killSession(sess, protocol.ReasonSessionKilled, true) | |
| 502 | + | }() | |
| 503 | + | close(start) | |
| 504 | + | wg.Wait() | |
| 505 | + | ||
| 506 | + | require.NoError(t, createErr, "attempt %d: creation racing final removal must be admitted", attempt) | |
| 507 | + | require.Equal(t, 1, sessionCount(d), "attempt %d: exactly the new session must remain", attempt) | |
| 508 | + | d.mu.Lock() | |
| 509 | + | closing := d.closing | |
| 510 | + | d.mu.Unlock() | |
| 511 | + | require.False(t, closing, "attempt %d: empty→occupied must not mark the daemon closing", attempt) | |
| 512 | + | ||
| 513 | + | _ = d.killSession(firstSession(d), protocol.ReasonSessionKilled, true) | |
| 514 | + | releaseFirst() | |
| 515 | + | releaseSecond() | |
| 516 | + | d.sessWg.Wait() | |
| 517 | + | d.waitNotifies() | |
| 518 | + | } | |
| 519 | + | } | |
| 520 | + | ||
| 521 | + | // TestRestoreAfterEmptyRegistryWorks proves a stopped named session can be | |
| 522 | + | // restored by a fresh attach once the registry has returned to empty: the | |
| 523 | + | // daemon kept serving and the inactive authority survived the final removal. | |
| 524 | + | func TestRestoreAfterEmptyRegistryWorks(t *testing.T) { | |
| 525 | + | firstPTY, releaseFirst := newBlockingPTY(t) | |
| 526 | + | defer releaseFirst() | |
| 527 | + | secondPTY, releaseSecond := newBlockingPTY(t) | |
| 528 | + | defer releaseSecond() | |
| 529 | + | d := newTestDaemon(t, newFactorySeq(t, firstPTY, secondPTY), stubClock{}) | |
| 530 | + | ||
| 531 | + | sess, err := createSessionForTest(d, "work", false, t.TempDir(), domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 532 | + | require.NoError(t, err) | |
| 533 | + | ||
| 534 | + | // Stop the only session; the registry returns to empty without shutdown. | |
| 535 | + | require.NoError(t, d.killSession(sess, protocol.ReasonServerShutdown, false)) | |
| 536 | + | require.Equal(t, 0, sessionCount(d)) | |
| 537 | + | select { | |
| 538 | + | case <-d.done: | |
| 539 | + | t.Fatal("stopped final session must not signal daemon shutdown") | |
| 540 | + | default: | |
| 541 | + | } | |
| 542 | + | ||
| 543 | + | // Restore it inside the same daemon. | |
| 544 | + | tr := newMockServerConnection(t) | |
| 545 | + | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | |
| 546 | + | tr.EXPECT().Close().Return(nil).Maybe() | |
| 547 | + | restored, ac, err := d.route(protocol.Hello{Version: protocol.Version, Intent: protocol.IntentAttach, Name: "work", Size: domain.Size{Cols: 80, Rows: 24}}, tr) | |
| 548 | + | require.NoError(t, err) | |
| 549 | + | require.NotNil(t, ac) | |
| 550 | + | require.Equal(t, "work", restored.name) | |
| 551 | + | require.Equal(t, 1, sessionCount(d)) | |
| 552 | + | ||
| 553 | + | _ = d.killSession(restored, protocol.ReasonServerShutdown, false) | |
| 411 | 554 | d.sessWg.Wait() | |
| 412 | 555 | d.waitNotifies() | |
| 413 | 556 | } | |
| 414 | 557 | ||
| 415 | 558 | // --- full Serve lifecycle --------------------------------------------------- | |
| 416 | 559 | ||
| 417 | - | func TestServeReturnsWhenLastSessionExits(t *testing.T) { | |
| 418 | - | p, releasePTY := newBlockingPTY(t) | |
| 419 | - | tr, sends, _ := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |
| 560 | + | // TestServeSurvivesEmptyRegistryAndAcceptsNewSession proves the accept loop and | |
| 561 | + | // listener are not torn down by an empty registry: after the last session | |
| 562 | + | // exits, the same Serve loop still admits a brand-new session, and only ctx | |
| 563 | + | // cancellation ends it. | |
| 564 | + | func TestServeSurvivesEmptyRegistryAndAcceptsNewSession(t *testing.T) { | |
| 565 | + | firstPTY, releaseFirst := newBlockingPTY(t) | |
| 566 | + | defer releaseFirst() | |
| 567 | + | secondPTY, releaseSecond := newBlockingPTY(t) | |
| 568 | + | defer releaseSecond() | |
| 569 | + | ||
| 570 | + | firstTr, firstSends, _ := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |
| 571 | + | secondTr, secondSends, _ := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |
| 420 | 572 | ||
| 421 | 573 | l := newMockServerListener(t) | |
| 574 | + | // Feed connections one at a time: the second is offered only after the | |
| 575 | + | // first session has exited, so the mock factory's PTY order is deterministic | |
| 576 | + | // and the reuse handshake proves the accept loop survived. | |
| 422 | 577 | connCh := make(chan wire.Transport, 1) | |
| 423 | - | connCh <- tr | |
| 578 | + | connCh <- firstTr | |
| 424 | 579 | closed := make(chan struct{}) | |
| 425 | 580 | var once sync.Once | |
| 426 | 581 | l.EXPECT().Accept().RunAndReturn(func() (wire.Transport, error) { | |
| @@ -434,22 +589,35 @@ func TestServeReturnsWhenLastSessionExits(t *testing.T) { | |||
| 434 | 589 | l.EXPECT().Close().RunAndReturn(func() error { once.Do(func() { close(closed) }); return nil }).Maybe() | |
| 435 | 590 | l.EXPECT().Addr().Return("mock").Maybe() | |
| 436 | 591 | ||
| 437 | - | d := New(newFactory(t, p), stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil))) | |
| 592 | + | d := New(newFactorySeq(t, firstPTY, secondPTY), stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil))) | |
| 593 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 594 | + | defer cancel() | |
| 438 | 595 | ||
| 439 | 596 | served := make(chan error, 1) | |
| 440 | - | go func() { served <- d.Serve(context.Background(), l) }() | |
| 597 | + | go func() { served <- d.Serve(ctx, l) }() | |
| 441 | 598 | ||
| 442 | - | awaitFrame(t, sends, "Welcome") | |
| 443 | - | awaitFrame(t, sends, "Output") | |
| 599 | + | awaitFrame(t, firstSends, "Welcome") | |
| 600 | + | awaitFrame(t, firstSends, "Output") | |
| 444 | 601 | ||
| 445 | - | // Child exits -> session removed -> registry empties -> Serve returns. | |
| 446 | - | releasePTY() | |
| 602 | + | // Child exits -> session removed -> registry empties -> Serve must survive. | |
| 603 | + | releaseFirst() | |
| 604 | + | det := awaitFrame(t, firstSends, "Detached") | |
| 605 | + | require.Equal(t, protocol.ReasonSessionKilled, decodeServerMessage(t, det).(protocol.Detached).Reason) | |
| 606 | + | require.Equal(t, 0, sessionCount(d)) | |
| 607 | + | ||
| 608 | + | // The same accept loop admits a fresh connection into the empty registry. | |
| 609 | + | connCh <- secondTr | |
| 610 | + | awaitFrame(t, secondSends, "Welcome") | |
| 611 | + | awaitFrame(t, secondSends, "Output") | |
| 612 | + | require.Equal(t, 1, sessionCount(d)) | |
| 447 | 613 | ||
| 614 | + | // Only explicit cancellation stops Serve. | |
| 615 | + | cancel() | |
| 448 | 616 | select { | |
| 449 | 617 | case err := <-served: | |
| 450 | 618 | require.NoError(t, err) | |
| 451 | 619 | case <-time.After(3 * time.Second): | |
| 452 | - | t.Fatal("Serve did not return after last session exited") | |
| 620 | + | t.Fatal("Serve did not return after context cancel") | |
| 453 | 621 | } | |
| 454 | 622 | } | |
| 455 | 623 | ||
| @@ -499,22 +667,18 @@ func TestServeGracefulShutdownOnContextCancel(t *testing.T) { | |||
| 499 | 667 | ||
| 500 | 668 | // --- shutdown/create interlock ------------------------------------------------ | |
| 501 | 669 | ||
| 502 | - | // TestHelloRacingShutdownIsRejected covers the interlock between the | |
| 503 | - | // registry-empty shutdown decision and session creation. In particular, the | |
| 504 | - | // old `.Once()` factory expectation was wrong: firstPaint legitimately queues | |
| 505 | - | // a floating prewarm, so an Open that starts before killSession is valid work, | |
| 506 | - | // not a leaked post-shutdown child. The factory markers classify timing at the | |
| 507 | - | // Open boundary: this test first observes that valid prewarm, then verifies | |
| 508 | - | // that d.done (closed synchronously by killSession) is never followed by an | |
| 509 | - | // Open start. | |
| 510 | - | func TestHelloRacingShutdownIsRejected(t *testing.T) { | |
| 670 | + | // TestHelloAfterExplicitShutdownIsRejected covers the shutdown/create | |
| 671 | + | // interlock that survives P4.1: an explicit shutdownAll still marks the daemon | |
| 672 | + | // closing synchronously and signals Serve, so Hellos arriving afterward are | |
| 673 | + | // rejected with a clean typed error instead of creating a session the teardown | |
| 674 | + | // snapshot missed. Final session removal alone no longer triggers this. | |
| 675 | + | func TestHelloAfterExplicitShutdownIsRejected(t *testing.T) { | |
| 511 | 676 | p, releasePTY := newBlockingPTY(t) | |
| 512 | 677 | floating := newQuietPTY() | |
| 513 | 678 | f := portsmocks.NewMockPTYFactory(t) | |
| 514 | 679 | normalSize := domain.Size{Cols: 80, Rows: 22} | |
| 515 | - | preShutdownFloatingOpen := make(chan struct{}, 1) | |
| 680 | + | var d *Daemon | |
| 516 | 681 | var opensAfterShutdown atomic.Int32 | |
| 517 | - | d := newTestDaemon(t, f, stubClock{}) | |
| 518 | 682 | f.EXPECT().Open(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).RunAndReturn( | |
| 519 | 683 | func(_ context.Context, _ string, _ []string, _ []string, _ string, geometry domain.Geometry) (ports.PTY, error) { | |
| 520 | 684 | select { | |
| @@ -525,40 +689,29 @@ func TestHelloRacingShutdownIsRejected(t *testing.T) { | |||
| 525 | 689 | if geometry.Size == normalSize { | |
| 526 | 690 | return p, nil | |
| 527 | 691 | } | |
| 528 | - | select { | |
| 529 | - | case preShutdownFloatingOpen <- struct{}{}: | |
| 530 | - | default: | |
| 531 | - | } | |
| 532 | 692 | return floating, nil | |
| 533 | 693 | }, | |
| 534 | 694 | ).Maybe() | |
| 695 | + | d = newTestDaemon(t, f, stubClock{}) | |
| 535 | 696 | ||
| 536 | 697 | tr1, sends1, release1 := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |
| 537 | 698 | var hg sync.WaitGroup | |
| 538 | 699 | hg.Go(func() { d.handleConn(tr1) }) | |
| 539 | 700 | awaitFrame(t, sends1, "Welcome") | |
| 540 | - | // firstPaint starts the background prewarm asynchronously. It is valid for | |
| 541 | - | // this Open to occur before shutdown, so do not mistake it for the racing | |
| 542 | - | // Hello's forbidden launch. | |
| 543 | 701 | awaitFrame(t, sends1, "Output") | |
| 544 | - | select { | |
| 545 | - | case <-preShutdownFloatingOpen: | |
| 546 | - | case <-time.After(time.Second): | |
| 547 | - | t.Fatal("legitimate pre-shutdown floating prewarm did not start") | |
| 548 | - | } | |
| 549 | - | sess := firstSession(d) | |
| 550 | - | require.NotNil(t, sess) | |
| 702 | + | require.Equal(t, 1, sessionCount(d)) | |
| 551 | 703 | ||
| 552 | - | // The last session dies: shutdown begins irreversibly. | |
| 553 | - | _ = d.killSession(sess, protocol.ReasonSessionKilled, false) | |
| 704 | + | // Explicit shutdown begins irreversibly and signals Serve synchronously. | |
| 705 | + | require.False(t, d.shutdownAll(protocol.ReasonServerShutdown)) | |
| 554 | 706 | select { | |
| 555 | 707 | case <-d.done: | |
| 556 | 708 | default: | |
| 557 | - | t.Fatal("registry-empty shutdown must be signalled synchronously by killSession") | |
| 709 | + | t.Fatal("explicit shutdown must signal Serve synchronously") | |
| 558 | 710 | } | |
| 711 | + | require.Equal(t, 0, sessionCount(d)) | |
| 559 | 712 | ||
| 560 | - | // Racing Hellos — both creation intents must be rejected with a clean | |
| 561 | - | // typed error, and nothing may be inserted into the registry. | |
| 713 | + | // Hellos after shutdown — both creation intents — must be rejected with a | |
| 714 | + | // clean typed error, and nothing may be inserted into the registry. | |
| 562 | 715 | for _, intent := range []struct { | |
| 563 | 716 | name string | |
| 564 | 717 | intent uint8 | |
| @@ -572,17 +725,16 @@ func TestHelloRacingShutdownIsRejected(t *testing.T) { | |||
| 572 | 725 | e := awaitFrame(t, sends2, "Error") | |
| 573 | 726 | em, ok := decodeServerMessage(t, e).(protocol.ErrorMsg) | |
| 574 | 727 | require.True(t, ok) | |
| 575 | - | require.Equal(t, protocol.ErrServerShutdown, em.Code, "%s hello racing shutdown must be rejected", intent.name) | |
| 728 | + | require.Equal(t, protocol.ErrServerShutdown, em.Code, "%s hello after shutdown must be rejected", intent.name) | |
| 576 | 729 | } | |
| 577 | 730 | require.Equal(t, 0, sessionCount(d), "no session may be inserted after shutdown began") | |
| 578 | - | require.Zero(t, opensAfterShutdown.Load(), "no PTYFactory.Open may start after d.done/shutdown completion") | |
| 579 | 731 | ||
| 580 | 732 | release1() | |
| 581 | 733 | releasePTY() | |
| 582 | 734 | hg.Wait() | |
| 583 | 735 | d.sessWg.Wait() | |
| 584 | 736 | d.waitNotifies() | |
| 585 | - | require.Zero(t, opensAfterShutdown.Load(), "racing Hellos must not start a PTY after shutdown") | |
| 737 | + | require.Zero(t, opensAfterShutdown.Load(), "shutdown must not be followed by a PTY launch") | |
| 586 | 738 | } | |
| 587 | 739 | ||
| 588 | 740 | // --- wedged-client teardown --------------------------------------------------- | |
| @@ -136,16 +136,51 @@ func TestHandshakeNoSuchSession(t *testing.T) { | |||||
| 136 | require.Equal(t, protocol.ErrNoSuchSession, em.Code) | 136 | require.Equal(t, protocol.ErrNoSuchSession, em.Code) | ||
| 137 | } | 137 | } | ||
| 138 | 138 | ||||
| 139 | - | func TestKillAllEmptyDaemonSignalsShutdown(t *testing.T) { | 139 | + | // TestKillAllEmptyDaemonKeepsServing proves KillAll no longer ends the daemon: |
| 140 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 140 | + | // purging an empty registry leaves closing, move admission, the daemon-wide |
| 141 | + | // pane process context, and d.done untouched, and the same daemon then admits a | |||
| 142 | + | // fresh session. | |||
| 143 | + | func TestKillAllEmptyDaemonKeepsServing(t *testing.T) { | |||
| 144 | + | pty, release := newBlockingPTY(t) | |||
| 145 | + | defer release() | |||
| 146 | + | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | |||
| 141 | tr, _, _ := newConn(t, mustClientEnvelope(protocol.Kill{Scope: protocol.KillAll})) | 147 | tr, _, _ := newConn(t, mustClientEnvelope(protocol.Kill{Scope: protocol.KillAll})) | ||
| 142 | d.handleConn(tr) | 148 | d.handleConn(tr) | ||
| 143 | 149 | ||||
| 144 | select { | 150 | select { | ||
| 145 | case <-d.done: | 151 | case <-d.done: | ||
| 146 | - | case <-time.After(time.Second): | 152 | + | t.Fatal("kill all on empty daemon must not signal shutdown") |
| 147 | - | t.Fatal("kill all on empty daemon did not signal shutdown") | 153 | + | default: |
| 148 | } | 154 | } | ||
| 155 | + | d.mu.Lock() | |||
| 156 | + | closing := d.closing | |||
| 157 | + | d.mu.Unlock() | |||
| 158 | + | require.False(t, closing, "kill all must not mark the daemon closing") | |||
| 159 | + | d.moveLifecycleMu.Lock() | |||
| 160 | + | moveClosing := d.moveLifecycleClosing | |||
| 161 | + | d.moveLifecycleMu.Unlock() | |||
| 162 | + | require.False(t, moveClosing, "kill all must not close move admission") | |||
| 163 | + | select { | |||
| 164 | + | case <-d.paneProcessCtx.Done(): | |||
| 165 | + | t.Fatal("kill all must not cancel the daemon-wide pane process context") | |||
| 166 | + | default: | |||
| 167 | + | } | |||
| 168 | + | ||||
| 169 | + | // The purge reopened admission: the same daemon creates a fresh session. | |||
| 170 | + | tr2, sends2, releaseConn2 := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |||
| 171 | + | var hg sync.WaitGroup | |||
| 172 | + | hg.Go(func() { d.handleConn(tr2) }) | |||
| 173 | + | welcome := decodeServerMessage(t, awaitFrame(t, sends2, "Welcome")).(protocol.Welcome) | |||
| 174 | + | require.Equal(t, "0", welcome.SessionName) | |||
| 175 | + | awaitFrame(t, sends2, "Output") | |||
| 176 | + | require.Equal(t, 1, sessionCount(d)) | |||
| 177 | + | ||||
| 178 | + | releaseConn2() | |||
| 179 | + | _ = d.killSession(firstSession(d), protocol.ReasonSessionKilled, true) | |||
| 180 | + | release() | |||
| 181 | + | hg.Wait() | |||
| 182 | + | d.sessWg.Wait() | |||
| 183 | + | d.waitNotifies() | |||
| 149 | } | 184 | } | ||
| 150 | 185 | ||||
| 151 | func TestBeginTeardownRequestResolvesShutdownPreservation(t *testing.T) { | 186 | func TestBeginTeardownRequestResolvesShutdownPreservation(t *testing.T) { | ||
| @@ -382,14 +417,21 @@ func TestDetachKeepsNamed(t *testing.T) { | |||||
| 382 | 417 | ||||
| 383 | // --- render scheduler debounce ---------------------------------------------- | 418 | // --- render scheduler debounce ---------------------------------------------- | ||
| 384 | 419 | ||||
| 385 | - | func TestReaderEOFRemovesSessionAndSignalsShutdown(t *testing.T) { | 420 | + | // TestReaderEOFRemovesSessionAndDaemonSurvives proves final session removal is |
| 386 | - | p := portsmocks.NewMockPTY(t) | 421 | + | // independent of daemon shutdown: the last session's EOF empties the registry |
| 422 | + | // without signalling shutdown or marking the daemon closing, and the same | |||
| 423 | + | // daemon can be occupied again by a fresh session. | |||
| 424 | + | func TestReaderEOFRemovesSessionAndDaemonSurvives(t *testing.T) { | |||
| 425 | + | eofPTY := portsmocks.NewMockPTY(t) | |||
| 387 | // Read returns EOF immediately (child already gone). | 426 | // Read returns EOF immediately (child already gone). | ||
| 388 | - | p.EXPECT().Read(mock.Anything).Return(0, io.EOF).Maybe() | 427 | + | eofPTY.EXPECT().Read(mock.Anything).Return(0, io.EOF).Maybe() |
| 389 | - | p.EXPECT().Close().Return(nil).Maybe() | 428 | + | eofPTY.EXPECT().Close().Return(nil).Maybe() |
| 390 | - | p.EXPECT().Pid().Return(1).Maybe() | 429 | + | eofPTY.EXPECT().Pid().Return(1).Maybe() |
| 430 | + | ||||
| 431 | + | secondPTY, releaseSecond := newBlockingPTY(t) | |||
| 432 | + | defer releaseSecond() | |||
| 433 | + | d := newTestDaemon(t, newFactorySeq(t, eofPTY, secondPTY), stubClock{}) | |||
| 391 | 434 | ||||
| 392 | - | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |||
| 393 | tr, sends, _ := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | 435 | tr, sends, _ := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | ||
| 394 | 436 | ||||
| 395 | var hg sync.WaitGroup | 437 | var hg sync.WaitGroup | ||
| @@ -400,27 +442,140 @@ func TestReaderEOFRemovesSessionAndSignalsShutdown(t *testing.T) { | |||||
| 400 | dm := decodeServerMessage(t, det).(protocol.Detached) | 442 | dm := decodeServerMessage(t, det).(protocol.Detached) | ||
| 401 | require.Equal(t, protocol.ReasonSessionKilled, dm.Reason) | 443 | require.Equal(t, protocol.ReasonSessionKilled, dm.Reason) | ||
| 402 | 444 | ||||
| 445 | + | // The registry is empty, yet the daemon has not begun shutdown and has not | |||
| 446 | + | // signalled Serve. The Detached notice is scheduled only after registry | |||
| 447 | + | // removal has published, so these assertions are deterministic. | |||
| 403 | select { | 448 | select { | ||
| 404 | case <-d.done: | 449 | case <-d.done: | ||
| 405 | - | case <-time.After(2 * time.Second): | 450 | + | t.Fatal("final session removal must not signal daemon shutdown") |
| 406 | - | t.Fatal("registry-empty shutdown was not signalled") | 451 | + | default: |
| 407 | } | 452 | } | ||
| 453 | + | d.mu.Lock() | |||
| 454 | + | closing := d.closing | |||
| 455 | + | d.mu.Unlock() | |||
| 456 | + | require.False(t, closing, "empty registry must not mark the daemon closing") | |||
| 408 | require.Equal(t, 0, sessionCount(d)) | 457 | require.Equal(t, 0, sessionCount(d)) | ||
| 409 | 458 | ||||
| 459 | + | // The empty registry can be occupied again by a brand-new session. | |||
| 460 | + | tr2, sends2, releaseConn2 := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |||
| 461 | + | var hg2 sync.WaitGroup | |||
| 462 | + | hg2.Go(func() { d.handleConn(tr2) }) | |||
| 463 | + | welcome := decodeServerMessage(t, awaitFrame(t, sends2, "Welcome")).(protocol.Welcome) | |||
| 464 | + | require.Equal(t, "0", welcome.SessionName) | |||
| 465 | + | awaitFrame(t, sends2, "Output") | |||
| 466 | + | require.Equal(t, 1, sessionCount(d)) | |||
| 467 | + | ||||
| 468 | + | _ = d.killSession(firstSession(d), protocol.ReasonSessionKilled, true) | |||
| 469 | + | releaseConn2() | |||
| 410 | hg.Wait() | 470 | hg.Wait() | ||
| 471 | + | hg2.Wait() | |||
| 472 | + | d.sessWg.Wait() | |||
| 473 | + | d.waitNotifies() | |||
| 474 | + | } | |||
| 475 | + | ||||
| 476 | + | // TestConcurrentFinalRemovalAndCreationLinearize proves final-session removal | |||
| 477 | + | // no longer opens a shutdown window: a create racing the removal is always | |||
| 478 | + | // admitted (never rejected with ErrServerShutdown) and the two registry | |||
| 479 | + | // transitions linearize under d.mu, leaving exactly the newly created session. | |||
| 480 | + | func TestConcurrentFinalRemovalAndCreationLinearize(t *testing.T) { | |||
| 481 | + | for attempt := 0; attempt < 32; attempt++ { | |||
| 482 | + | firstPTY, releaseFirst := newBlockingPTY(t) | |||
| 483 | + | secondPTY, releaseSecond := newBlockingPTY(t) | |||
| 484 | + | d := newTestDaemon(t, newFactorySeq(t, firstPTY, secondPTY), stubClock{}) | |||
| 485 | + | ||||
| 486 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, nil) | |||
| 487 | + | require.NoError(t, err) | |||
| 488 | + | ||||
| 489 | + | start := make(chan struct{}) | |||
| 490 | + | var wg sync.WaitGroup | |||
| 491 | + | wg.Add(2) | |||
| 492 | + | var createErr error | |||
| 493 | + | go func() { | |||
| 494 | + | defer wg.Done() | |||
| 495 | + | <-start | |||
| 496 | + | _, _, createErr = d.route(helloResumeCapable(protocol.IntentEphemeral, "", 0), &closeTrackingTransport{}) | |||
| 497 | + | }() | |||
| 498 | + | go func() { | |||
| 499 | + | defer wg.Done() | |||
| 500 | + | <-start | |||
| 501 | + | _ = d.killSession(sess, protocol.ReasonSessionKilled, true) | |||
| 502 | + | }() | |||
| 503 | + | close(start) | |||
| 504 | + | wg.Wait() | |||
| 505 | + | ||||
| 506 | + | require.NoError(t, createErr, "attempt %d: creation racing final removal must be admitted", attempt) | |||
| 507 | + | require.Equal(t, 1, sessionCount(d), "attempt %d: exactly the new session must remain", attempt) | |||
| 508 | + | d.mu.Lock() | |||
| 509 | + | closing := d.closing | |||
| 510 | + | d.mu.Unlock() | |||
| 511 | + | require.False(t, closing, "attempt %d: empty→occupied must not mark the daemon closing", attempt) | |||
| 512 | + | ||||
| 513 | + | _ = d.killSession(firstSession(d), protocol.ReasonSessionKilled, true) | |||
| 514 | + | releaseFirst() | |||
| 515 | + | releaseSecond() | |||
| 516 | + | d.sessWg.Wait() | |||
| 517 | + | d.waitNotifies() | |||
| 518 | + | } | |||
| 519 | + | } | |||
| 520 | + | ||||
| 521 | + | // TestRestoreAfterEmptyRegistryWorks proves a stopped named session can be | |||
| 522 | + | // restored by a fresh attach once the registry has returned to empty: the | |||
| 523 | + | // daemon kept serving and the inactive authority survived the final removal. | |||
| 524 | + | func TestRestoreAfterEmptyRegistryWorks(t *testing.T) { | |||
| 525 | + | firstPTY, releaseFirst := newBlockingPTY(t) | |||
| 526 | + | defer releaseFirst() | |||
| 527 | + | secondPTY, releaseSecond := newBlockingPTY(t) | |||
| 528 | + | defer releaseSecond() | |||
| 529 | + | d := newTestDaemon(t, newFactorySeq(t, firstPTY, secondPTY), stubClock{}) | |||
| 530 | + | ||||
| 531 | + | sess, err := createSessionForTest(d, "work", false, t.TempDir(), domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |||
| 532 | + | require.NoError(t, err) | |||
| 533 | + | ||||
| 534 | + | // Stop the only session; the registry returns to empty without shutdown. | |||
| 535 | + | require.NoError(t, d.killSession(sess, protocol.ReasonServerShutdown, false)) | |||
| 536 | + | require.Equal(t, 0, sessionCount(d)) | |||
| 537 | + | select { | |||
| 538 | + | case <-d.done: | |||
| 539 | + | t.Fatal("stopped final session must not signal daemon shutdown") | |||
| 540 | + | default: | |||
| 541 | + | } | |||
| 542 | + | ||||
| 543 | + | // Restore it inside the same daemon. | |||
| 544 | + | tr := newMockServerConnection(t) | |||
| 545 | + | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | |||
| 546 | + | tr.EXPECT().Close().Return(nil).Maybe() | |||
| 547 | + | restored, ac, err := d.route(protocol.Hello{Version: protocol.Version, Intent: protocol.IntentAttach, Name: "work", Size: domain.Size{Cols: 80, Rows: 24}}, tr) | |||
| 548 | + | require.NoError(t, err) | |||
| 549 | + | require.NotNil(t, ac) | |||
| 550 | + | require.Equal(t, "work", restored.name) | |||
| 551 | + | require.Equal(t, 1, sessionCount(d)) | |||
| 552 | + | ||||
| 553 | + | _ = d.killSession(restored, protocol.ReasonServerShutdown, false) | |||
| 411 | d.sessWg.Wait() | 554 | d.sessWg.Wait() | ||
| 412 | d.waitNotifies() | 555 | d.waitNotifies() | ||
| 413 | } | 556 | } | ||
| 414 | 557 | ||||
| 415 | // --- full Serve lifecycle --------------------------------------------------- | 558 | // --- full Serve lifecycle --------------------------------------------------- | ||
| 416 | 559 | ||||
| 417 | - | func TestServeReturnsWhenLastSessionExits(t *testing.T) { | 560 | + | // TestServeSurvivesEmptyRegistryAndAcceptsNewSession proves the accept loop and |
| 418 | - | p, releasePTY := newBlockingPTY(t) | 561 | + | // listener are not torn down by an empty registry: after the last session |
| 419 | - | tr, sends, _ := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | 562 | + | // exits, the same Serve loop still admits a brand-new session, and only ctx |
| 563 | + | // cancellation ends it. | |||
| 564 | + | func TestServeSurvivesEmptyRegistryAndAcceptsNewSession(t *testing.T) { | |||
| 565 | + | firstPTY, releaseFirst := newBlockingPTY(t) | |||
| 566 | + | defer releaseFirst() | |||
| 567 | + | secondPTY, releaseSecond := newBlockingPTY(t) | |||
| 568 | + | defer releaseSecond() | |||
| 569 | + | ||||
| 570 | + | firstTr, firstSends, _ := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |||
| 571 | + | secondTr, secondSends, _ := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | |||
| 420 | 572 | ||||
| 421 | l := newMockServerListener(t) | 573 | l := newMockServerListener(t) | ||
| 574 | + | // Feed connections one at a time: the second is offered only after the | |||
| 575 | + | // first session has exited, so the mock factory's PTY order is deterministic | |||
| 576 | + | // and the reuse handshake proves the accept loop survived. | |||
| 422 | connCh := make(chan wire.Transport, 1) | 577 | connCh := make(chan wire.Transport, 1) | ||
| 423 | - | connCh <- tr | 578 | + | connCh <- firstTr |
| 424 | closed := make(chan struct{}) | 579 | closed := make(chan struct{}) | ||
| 425 | var once sync.Once | 580 | var once sync.Once | ||
| 426 | l.EXPECT().Accept().RunAndReturn(func() (wire.Transport, error) { | 581 | l.EXPECT().Accept().RunAndReturn(func() (wire.Transport, error) { | ||
| @@ -434,22 +589,35 @@ func TestServeReturnsWhenLastSessionExits(t *testing.T) { | |||||
| 434 | l.EXPECT().Close().RunAndReturn(func() error { once.Do(func() { close(closed) }); return nil }).Maybe() | 589 | l.EXPECT().Close().RunAndReturn(func() error { once.Do(func() { close(closed) }); return nil }).Maybe() | ||
| 435 | l.EXPECT().Addr().Return("mock").Maybe() | 590 | l.EXPECT().Addr().Return("mock").Maybe() | ||
| 436 | 591 | ||||
| 437 | - | d := New(newFactory(t, p), stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil))) | 592 | + | d := New(newFactorySeq(t, firstPTY, secondPTY), stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil))) |
| 593 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 594 | + | defer cancel() | |||
| 438 | 595 | ||||
| 439 | served := make(chan error, 1) | 596 | served := make(chan error, 1) | ||
| 440 | - | go func() { served <- d.Serve(context.Background(), l) }() | 597 | + | go func() { served <- d.Serve(ctx, l) }() |
| 441 | 598 | ||||
| 442 | - | awaitFrame(t, sends, "Welcome") | 599 | + | awaitFrame(t, firstSends, "Welcome") |
| 443 | - | awaitFrame(t, sends, "Output") | 600 | + | awaitFrame(t, firstSends, "Output") |
| 444 | 601 | ||||
| 445 | - | // Child exits -> session removed -> registry empties -> Serve returns. | 602 | + | // Child exits -> session removed -> registry empties -> Serve must survive. |
| 446 | - | releasePTY() | 603 | + | releaseFirst() |
| 604 | + | det := awaitFrame(t, firstSends, "Detached") | |||
| 605 | + | require.Equal(t, protocol.ReasonSessionKilled, decodeServerMessage(t, det).(protocol.Detached).Reason) | |||
| 606 | + | require.Equal(t, 0, sessionCount(d)) | |||
| 607 | + | ||||
| 608 | + | // The same accept loop admits a fresh connection into the empty registry. | |||
| 609 | + | connCh <- secondTr | |||
| 610 | + | awaitFrame(t, secondSends, "Welcome") | |||
| 611 | + | awaitFrame(t, secondSends, "Output") | |||
| 612 | + | require.Equal(t, 1, sessionCount(d)) | |||
| 447 | 613 | ||||
| 614 | + | // Only explicit cancellation stops Serve. | |||
| 615 | + | cancel() | |||
| 448 | select { | 616 | select { | ||
| 449 | case err := <-served: | 617 | case err := <-served: | ||
| 450 | require.NoError(t, err) | 618 | require.NoError(t, err) | ||
| 451 | case <-time.After(3 * time.Second): | 619 | case <-time.After(3 * time.Second): | ||
| 452 | - | t.Fatal("Serve did not return after last session exited") | 620 | + | t.Fatal("Serve did not return after context cancel") |
| 453 | } | 621 | } | ||
| 454 | } | 622 | } | ||
| 455 | 623 | ||||
| @@ -499,22 +667,18 @@ func TestServeGracefulShutdownOnContextCancel(t *testing.T) { | |||||
| 499 | 667 | ||||
| 500 | // --- shutdown/create interlock ------------------------------------------------ | 668 | // --- shutdown/create interlock ------------------------------------------------ | ||
| 501 | 669 | ||||
| 502 | - | // TestHelloRacingShutdownIsRejected covers the interlock between the | 670 | + | // TestHelloAfterExplicitShutdownIsRejected covers the shutdown/create |
| 503 | - | // registry-empty shutdown decision and session creation. In particular, the | 671 | + | // interlock that survives P4.1: an explicit shutdownAll still marks the daemon |
| 504 | - | // old `.Once()` factory expectation was wrong: firstPaint legitimately queues | 672 | + | // closing synchronously and signals Serve, so Hellos arriving afterward are |
| 505 | - | // a floating prewarm, so an Open that starts before killSession is valid work, | 673 | + | // rejected with a clean typed error instead of creating a session the teardown |
| 506 | - | // not a leaked post-shutdown child. The factory markers classify timing at the | 674 | + | // snapshot missed. Final session removal alone no longer triggers this. |
| 507 | - | // Open boundary: this test first observes that valid prewarm, then verifies | 675 | + | func TestHelloAfterExplicitShutdownIsRejected(t *testing.T) { |
| 508 | - | // that d.done (closed synchronously by killSession) is never followed by an | |||
| 509 | - | // Open start. | |||
| 510 | - | func TestHelloRacingShutdownIsRejected(t *testing.T) { | |||
| 511 | p, releasePTY := newBlockingPTY(t) | 676 | p, releasePTY := newBlockingPTY(t) | ||
| 512 | floating := newQuietPTY() | 677 | floating := newQuietPTY() | ||
| 513 | f := portsmocks.NewMockPTYFactory(t) | 678 | f := portsmocks.NewMockPTYFactory(t) | ||
| 514 | normalSize := domain.Size{Cols: 80, Rows: 22} | 679 | normalSize := domain.Size{Cols: 80, Rows: 22} | ||
| 515 | - | preShutdownFloatingOpen := make(chan struct{}, 1) | 680 | + | var d *Daemon |
| 516 | var opensAfterShutdown atomic.Int32 | 681 | var opensAfterShutdown atomic.Int32 | ||
| 517 | - | d := newTestDaemon(t, f, stubClock{}) | |||
| 518 | f.EXPECT().Open(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).RunAndReturn( | 682 | f.EXPECT().Open(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).RunAndReturn( | ||
| 519 | func(_ context.Context, _ string, _ []string, _ []string, _ string, geometry domain.Geometry) (ports.PTY, error) { | 683 | func(_ context.Context, _ string, _ []string, _ []string, _ string, geometry domain.Geometry) (ports.PTY, error) { | ||
| 520 | select { | 684 | select { | ||
| @@ -525,40 +689,29 @@ func TestHelloRacingShutdownIsRejected(t *testing.T) { | |||||
| 525 | if geometry.Size == normalSize { | 689 | if geometry.Size == normalSize { | ||
| 526 | return p, nil | 690 | return p, nil | ||
| 527 | } | 691 | } | ||
| 528 | - | select { | |||
| 529 | - | case preShutdownFloatingOpen <- struct{}{}: | |||
| 530 | - | default: | |||
| 531 | - | } | |||
| 532 | return floating, nil | 692 | return floating, nil | ||
| 533 | }, | 693 | }, | ||
| 534 | ).Maybe() | 694 | ).Maybe() | ||
| 695 | + | d = newTestDaemon(t, f, stubClock{}) | |||
| 535 | 696 | ||||
| 536 | tr1, sends1, release1 := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | 697 | tr1, sends1, release1 := newConn(t, mustHello(protocol.IntentEphemeral, "", domain.Size{Cols: 80, Rows: 24})) | ||
| 537 | var hg sync.WaitGroup | 698 | var hg sync.WaitGroup | ||
| 538 | hg.Go(func() { d.handleConn(tr1) }) | 699 | hg.Go(func() { d.handleConn(tr1) }) | ||
| 539 | awaitFrame(t, sends1, "Welcome") | 700 | awaitFrame(t, sends1, "Welcome") | ||
| 540 | - | // firstPaint starts the background prewarm asynchronously. It is valid for | |||
| 541 | - | // this Open to occur before shutdown, so do not mistake it for the racing | |||
| 542 | - | // Hello's forbidden launch. | |||
| 543 | awaitFrame(t, sends1, "Output") | 701 | awaitFrame(t, sends1, "Output") | ||
| 544 | - | select { | 702 | + | require.Equal(t, 1, sessionCount(d)) |
| 545 | - | case <-preShutdownFloatingOpen: | |||
| 546 | - | case <-time.After(time.Second): | |||
| 547 | - | t.Fatal("legitimate pre-shutdown floating prewarm did not start") | |||
| 548 | - | } | |||
| 549 | - | sess := firstSession(d) | |||
| 550 | - | require.NotNil(t, sess) | |||
| 551 | 703 | ||||
| 552 | - | // The last session dies: shutdown begins irreversibly. | 704 | + | // Explicit shutdown begins irreversibly and signals Serve synchronously. |
| 553 | - | _ = d.killSession(sess, protocol.ReasonSessionKilled, false) | 705 | + | require.False(t, d.shutdownAll(protocol.ReasonServerShutdown)) |
| 554 | select { | 706 | select { | ||
| 555 | case <-d.done: | 707 | case <-d.done: | ||
| 556 | default: | 708 | default: | ||
| 557 | - | t.Fatal("registry-empty shutdown must be signalled synchronously by killSession") | 709 | + | t.Fatal("explicit shutdown must signal Serve synchronously") |
| 558 | } | 710 | } | ||
| 711 | + | require.Equal(t, 0, sessionCount(d)) | |||
| 559 | 712 | ||||
| 560 | - | // Racing Hellos — both creation intents must be rejected with a clean | 713 | + | // Hellos after shutdown — both creation intents — must be rejected with a |
| 561 | - | // typed error, and nothing may be inserted into the registry. | 714 | + | // clean typed error, and nothing may be inserted into the registry. |
| 562 | for _, intent := range []struct { | 715 | for _, intent := range []struct { | ||
| 563 | name string | 716 | name string | ||
| 564 | intent uint8 | 717 | intent uint8 | ||
| @@ -572,17 +725,16 @@ func TestHelloRacingShutdownIsRejected(t *testing.T) { | |||||
| 572 | e := awaitFrame(t, sends2, "Error") | 725 | e := awaitFrame(t, sends2, "Error") | ||
| 573 | em, ok := decodeServerMessage(t, e).(protocol.ErrorMsg) | 726 | em, ok := decodeServerMessage(t, e).(protocol.ErrorMsg) | ||
| 574 | require.True(t, ok) | 727 | require.True(t, ok) | ||
| 575 | - | require.Equal(t, protocol.ErrServerShutdown, em.Code, "%s hello racing shutdown must be rejected", intent.name) | 728 | + | require.Equal(t, protocol.ErrServerShutdown, em.Code, "%s hello after shutdown must be rejected", intent.name) |
| 576 | } | 729 | } | ||
| 577 | require.Equal(t, 0, sessionCount(d), "no session may be inserted after shutdown began") | 730 | require.Equal(t, 0, sessionCount(d), "no session may be inserted after shutdown began") | ||
| 578 | - | require.Zero(t, opensAfterShutdown.Load(), "no PTYFactory.Open may start after d.done/shutdown completion") | |||
| 579 | 731 | ||||
| 580 | release1() | 732 | release1() | ||
| 581 | releasePTY() | 733 | releasePTY() | ||
| 582 | hg.Wait() | 734 | hg.Wait() | ||
| 583 | d.sessWg.Wait() | 735 | d.sessWg.Wait() | ||
| 584 | d.waitNotifies() | 736 | d.waitNotifies() | ||
| 585 | - | require.Zero(t, opensAfterShutdown.Load(), "racing Hellos must not start a PTY after shutdown") | 737 | + | require.Zero(t, opensAfterShutdown.Load(), "shutdown must not be followed by a PTY launch") |
| 586 | } | 738 | } | ||
| 587 | 739 | ||||
| 588 | // --- wedged-client teardown --------------------------------------------------- | 740 | // --- wedged-client teardown --------------------------------------------------- | ||
M internal/usecase/daemon/snapshot_coordinator.go +32 -1
| @@ -50,6 +50,32 @@ func quarantineSnapshotCoordinatorWithOptions(sess *session, retainQueuedCapture | |||
| 50 | 50 | return done | |
| 51 | 51 | } | |
| 52 | 52 | ||
| 53 | + | // rollbackSnapshotQuarantine reverses quarantineSnapshotCoordinator when a | |
| 54 | + | // teardown aborts before it owns any destructive repository operation and the | |
| 55 | + | // session therefore stays live and registered. The quarantine cancelled the | |
| 56 | + | // publication context and flipped eligibility, so a surviving session would | |
| 57 | + | // otherwise never checkpoint again: this discards the cancelled context so the | |
| 58 | + | // next schedule allocates a fresh one, restores the captured pre-quarantine | |
| 59 | + | // eligibility, and marks the session dirty so the mutations its ineligible fast | |
| 60 | + | // path suppressed during the quarantine window are captured by a later | |
| 61 | + | // publication. A queued capture the quarantine discarded stays discarded: | |
| 62 | + | // startSnapshotPublication rejects it by identity and the forced dirty | |
| 63 | + | // generation guarantees a replacement capture. | |
| 64 | + | func rollbackSnapshotQuarantine(sess *session, wasEligible bool) { | |
| 65 | + | if sess == nil { | |
| 66 | + | return | |
| 67 | + | } | |
| 68 | + | sess.snapshotMu.Lock() | |
| 69 | + | sess.snapshotQuarantined = false | |
| 70 | + | sess.snapshotPublicationContext = nil | |
| 71 | + | sess.snapshotPublicationCancel = nil | |
| 72 | + | sess.snapshotMu.Unlock() | |
| 73 | + | sess.snapEligible.Store(wasEligible) | |
| 74 | + | if wasEligible { | |
| 75 | + | markSnapshotDirty(sess) | |
| 76 | + | } | |
| 77 | + | } | |
| 78 | + | ||
| 53 | 79 | func markSnapshotDirty(sess *session) { | |
| 54 | 80 | if sess == nil || !sess.snapEligible.Load() { | |
| 55 | 81 | return | |
| @@ -160,7 +186,12 @@ func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) | |||
| 160 | 186 | capture.finishOnce.Do(func() { | |
| 161 | 187 | var shouldScheduleForcedSuccessor bool | |
| 162 | 188 | capture.session.snapshotMu.Lock() | |
| 163 | - | if capture.session.snapshotPendingCaptures > 0 { | |
| 189 | + | // A capture the quarantine discarded already dropped its pending count when | |
| 190 | + | // it was detached from the producer queue; the worker still calls finish on | |
| 191 | + | // it after rejecting the stale identity. Decrementing again here would | |
| 192 | + | // under-count a replacement capture scheduled after a rollback, so a | |
| 193 | + | // discarded capture must not decrement twice. | |
| 194 | + | if !capture.coordinatorDiscarded && capture.session.snapshotPendingCaptures > 0 { | |
| 164 | 195 | capture.session.snapshotPendingCaptures-- | |
| 165 | 196 | } | |
| 166 | 197 | if capture.session.snapshotQueuedCapture == capture { | |
| @@ -50,6 +50,32 @@ func quarantineSnapshotCoordinatorWithOptions(sess *session, retainQueuedCapture | |||||
| 50 | return done | 50 | return done | ||
| 51 | } | 51 | } | ||
| 52 | 52 | ||||
| 53 | + | // rollbackSnapshotQuarantine reverses quarantineSnapshotCoordinator when a | |||
| 54 | + | // teardown aborts before it owns any destructive repository operation and the | |||
| 55 | + | // session therefore stays live and registered. The quarantine cancelled the | |||
| 56 | + | // publication context and flipped eligibility, so a surviving session would | |||
| 57 | + | // otherwise never checkpoint again: this discards the cancelled context so the | |||
| 58 | + | // next schedule allocates a fresh one, restores the captured pre-quarantine | |||
| 59 | + | // eligibility, and marks the session dirty so the mutations its ineligible fast | |||
| 60 | + | // path suppressed during the quarantine window are captured by a later | |||
| 61 | + | // publication. A queued capture the quarantine discarded stays discarded: | |||
| 62 | + | // startSnapshotPublication rejects it by identity and the forced dirty | |||
| 63 | + | // generation guarantees a replacement capture. | |||
| 64 | + | func rollbackSnapshotQuarantine(sess *session, wasEligible bool) { | |||
| 65 | + | if sess == nil { | |||
| 66 | + | return | |||
| 67 | + | } | |||
| 68 | + | sess.snapshotMu.Lock() | |||
| 69 | + | sess.snapshotQuarantined = false | |||
| 70 | + | sess.snapshotPublicationContext = nil | |||
| 71 | + | sess.snapshotPublicationCancel = nil | |||
| 72 | + | sess.snapshotMu.Unlock() | |||
| 73 | + | sess.snapEligible.Store(wasEligible) | |||
| 74 | + | if wasEligible { | |||
| 75 | + | markSnapshotDirty(sess) | |||
| 76 | + | } | |||
| 77 | + | } | |||
| 78 | + | ||||
| 53 | func markSnapshotDirty(sess *session) { | 79 | func markSnapshotDirty(sess *session) { | ||
| 54 | if sess == nil || !sess.snapEligible.Load() { | 80 | if sess == nil || !sess.snapEligible.Load() { | ||
| 55 | return | 81 | return | ||
| @@ -160,7 +186,12 @@ func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) | |||||
| 160 | capture.finishOnce.Do(func() { | 186 | capture.finishOnce.Do(func() { | ||
| 161 | var shouldScheduleForcedSuccessor bool | 187 | var shouldScheduleForcedSuccessor bool | ||
| 162 | capture.session.snapshotMu.Lock() | 188 | capture.session.snapshotMu.Lock() | ||
| 163 | - | if capture.session.snapshotPendingCaptures > 0 { | 189 | + | // A capture the quarantine discarded already dropped its pending count when |
| 190 | + | // it was detached from the producer queue; the worker still calls finish on | |||
| 191 | + | // it after rejecting the stale identity. Decrementing again here would | |||
| 192 | + | // under-count a replacement capture scheduled after a rollback, so a | |||
| 193 | + | // discarded capture must not decrement twice. | |||
| 194 | + | if !capture.coordinatorDiscarded && capture.session.snapshotPendingCaptures > 0 { | |||
| 164 | capture.session.snapshotPendingCaptures-- | 195 | capture.session.snapshotPendingCaptures-- | ||
| 165 | } | 196 | } | ||
| 166 | if capture.session.snapshotQueuedCapture == capture { | 197 | if capture.session.snapshotQueuedCapture == capture { | ||
M internal/usecase/daemon/snapshot_restore.go +7 -4
| @@ -109,7 +109,7 @@ func validateRestoreSessionSnapshot(snap snapcodec.Session) error { | |||
| 109 | 109 | ||
| 110 | 110 | // restoreSession owns the restored tabs until registration succeeds. Every | |
| 111 | 111 | // unsuccessful path closes their PTYs and cancels their shared session context. | |
| 112 | - | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session, repositoryGeneration uint64, checkpoint domain.CheckpointRef) error { | |
| 112 | + | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session, repositoryGeneration uint64, checkpoint domain.CheckpointRef, epoch uint64) error { | |
| 113 | 113 | if err := ctx.Err(); err != nil { | |
| 114 | 114 | return err | |
| 115 | 115 | } | |
| @@ -147,7 +147,7 @@ func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session, rep | |||
| 147 | 147 | // checkpoints must continue from it rather than reuse generation one. | |
| 148 | 148 | sess.snapshotPublishedGeneration = repositoryGeneration | |
| 149 | 149 | sess.snapshotPublishedCheckpoint = &checkpoint | |
| 150 | - | registered, err := d.persistAndRegisterRestoredSession(ctx, sess) | |
| 150 | + | registered, err := d.persistAndRegisterRestoredSession(ctx, sess, epoch) | |
| 151 | 151 | if err != nil || !registered { | |
| 152 | 152 | return err | |
| 153 | 153 | } | |
| @@ -313,7 +313,10 @@ func (d *Daemon) newRestoredSession(snap snapcodec.Session, sctx context.Context | |||
| 313 | 313 | ||
| 314 | 314 | // persistAndRegisterRestoredSession atomically transfers a successfully | |
| 315 | 315 | // persisted session to the daemon. The caller retains cleanup on false/error. | |
| 316 | - | func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *session) (bool, error) { | |
| 316 | + | // The epoch revalidation makes publication a true admitted transition: if a | |
| 317 | + | // KillAll purge began and is waiting on this admitted restore, the session is | |
| 318 | + | // not published, so it cannot resurrect a record the purge is about to remove. | |
| 319 | + | func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *session, epoch uint64) (bool, error) { | |
| 317 | 320 | if sess.incarnation == (domain.IncarnationID{}) { | |
| 318 | 321 | d.mu.Lock() | |
| 319 | 322 | stopped := d.inactive[sess.name] | |
| @@ -343,7 +346,7 @@ func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *se | |||
| 343 | 346 | } | |
| 344 | 347 | d.mu.Lock() | |
| 345 | 348 | defer d.mu.Unlock() | |
| 346 | - | if d.closing || d.findByNameLocked(sess.name) != nil || ctx.Err() != nil { | |
| 349 | + | if d.purgeSupersededLocked(epoch) || d.findByNameLocked(sess.name) != nil || ctx.Err() != nil { | |
| 347 | 350 | return false, nil | |
| 348 | 351 | } | |
| 349 | 352 | sess.id = domain.SessionID(fmt.Sprintf("sess-%d", d.nextID)) | |
| @@ -109,7 +109,7 @@ func validateRestoreSessionSnapshot(snap snapcodec.Session) error { | |||||
| 109 | 109 | ||||
| 110 | // restoreSession owns the restored tabs until registration succeeds. Every | 110 | // restoreSession owns the restored tabs until registration succeeds. Every | ||
| 111 | // unsuccessful path closes their PTYs and cancels their shared session context. | 111 | // unsuccessful path closes their PTYs and cancels their shared session context. | ||
| 112 | - | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session, repositoryGeneration uint64, checkpoint domain.CheckpointRef) error { | 112 | + | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session, repositoryGeneration uint64, checkpoint domain.CheckpointRef, epoch uint64) error { |
| 113 | if err := ctx.Err(); err != nil { | 113 | if err := ctx.Err(); err != nil { | ||
| 114 | return err | 114 | return err | ||
| 115 | } | 115 | } | ||
| @@ -147,7 +147,7 @@ func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session, rep | |||||
| 147 | // checkpoints must continue from it rather than reuse generation one. | 147 | // checkpoints must continue from it rather than reuse generation one. | ||
| 148 | sess.snapshotPublishedGeneration = repositoryGeneration | 148 | sess.snapshotPublishedGeneration = repositoryGeneration | ||
| 149 | sess.snapshotPublishedCheckpoint = &checkpoint | 149 | sess.snapshotPublishedCheckpoint = &checkpoint | ||
| 150 | - | registered, err := d.persistAndRegisterRestoredSession(ctx, sess) | 150 | + | registered, err := d.persistAndRegisterRestoredSession(ctx, sess, epoch) |
| 151 | if err != nil || !registered { | 151 | if err != nil || !registered { | ||
| 152 | return err | 152 | return err | ||
| 153 | } | 153 | } | ||
| @@ -313,7 +313,10 @@ func (d *Daemon) newRestoredSession(snap snapcodec.Session, sctx context.Context | |||||
| 313 | 313 | ||||
| 314 | // persistAndRegisterRestoredSession atomically transfers a successfully | 314 | // persistAndRegisterRestoredSession atomically transfers a successfully | ||
| 315 | // persisted session to the daemon. The caller retains cleanup on false/error. | 315 | // persisted session to the daemon. The caller retains cleanup on false/error. | ||
| 316 | - | func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *session) (bool, error) { | 316 | + | // The epoch revalidation makes publication a true admitted transition: if a |
| 317 | + | // KillAll purge began and is waiting on this admitted restore, the session is | |||
| 318 | + | // not published, so it cannot resurrect a record the purge is about to remove. | |||
| 319 | + | func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *session, epoch uint64) (bool, error) { | |||
| 317 | if sess.incarnation == (domain.IncarnationID{}) { | 320 | if sess.incarnation == (domain.IncarnationID{}) { | ||
| 318 | d.mu.Lock() | 321 | d.mu.Lock() | ||
| 319 | stopped := d.inactive[sess.name] | 322 | stopped := d.inactive[sess.name] | ||
| @@ -343,7 +346,7 @@ func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *se | |||||
| 343 | } | 346 | } | ||
| 344 | d.mu.Lock() | 347 | d.mu.Lock() | ||
| 345 | defer d.mu.Unlock() | 348 | defer d.mu.Unlock() | ||
| 346 | - | if d.closing || d.findByNameLocked(sess.name) != nil || ctx.Err() != nil { | 349 | + | if d.purgeSupersededLocked(epoch) || d.findByNameLocked(sess.name) != nil || ctx.Err() != nil { |
| 347 | return false, nil | 350 | return false, nil | ||
| 348 | } | 351 | } | ||
| 349 | sess.id = domain.SessionID(fmt.Sprintf("sess-%d", d.nextID)) | 352 | sess.id = domain.SessionID(fmt.Sprintf("sess-%d", d.nextID)) | ||
M internal/usecase/daemon/snapshot_restore_incremental.go +62 -12
| @@ -39,26 +39,64 @@ func (d *Daemon) restoreIncrementalSnapshots(ctx context.Context) { | |||
| 39 | 39 | if d.catalogue == nil { | |
| 40 | 40 | return | |
| 41 | 41 | } | |
| 42 | + | records, epoch, ok := d.captureCatalogueSnapshot(ctx) | |
| 43 | + | if !ok { | |
| 44 | + | return | |
| 45 | + | } | |
| 46 | + | d.restoreCatalogueEpoch(ctx, records, epoch) | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | // captureCatalogueSnapshot reads the purge boundary and the authoritative | |
| 50 | + | // catalogue records together under purge admission. Holding the gate for both | |
| 51 | + | // reads keeps them atomic with any KillAll: a purge can never increment the | |
| 52 | + | // epoch between the two reads and pair a record set it already removed with a | |
| 53 | + | // boundary that looks current. The caller's context and the move/purge gates | |
| 54 | + | // bound the wait. | |
| 55 | + | func (d *Daemon) captureCatalogueSnapshot(ctx context.Context) ([]domain.CatalogueRecord, uint64, bool) { | |
| 56 | + | if err := d.acquirePurgeAdmission(ctx); err != nil { | |
| 57 | + | return nil, 0, false | |
| 58 | + | } | |
| 59 | + | defer d.releasePurgeAdmission() | |
| 60 | + | epoch := d.purgeEpochSnapshot() | |
| 42 | 61 | records, err := d.catalogue.Records() | |
| 43 | 62 | if err != nil { | |
| 44 | 63 | d.log.Error("loading catalogue for snapshot restoration failed", "err", err) | |
| 45 | - | return | |
| 64 | + | return nil, 0, false | |
| 46 | 65 | } | |
| 47 | - | d.restoreCatalogue(ctx, records) | |
| 66 | + | return records, epoch, true | |
| 48 | 67 | } | |
| 49 | 68 | ||
| 50 | - | // restoreCatalogue gives every expected record an independent bounded job. | |
| 51 | - | // A failed record is degraded without cancelling any sibling restoration. | |
| 52 | - | func (d *Daemon) restoreCatalogue(ctx context.Context, records []domain.CatalogueRecord) { | |
| 69 | + | // restoreCatalogueEpoch restores records captured at the given purge boundary. | |
| 70 | + | // Each worker holds purge admission for exactly one record's transition — its | |
| 71 | + | // registry publication, checkpoint load, and publication or demotion — so a | |
| 72 | + | // KillAll drains the in-flight record before it snapshots, and a worker that | |
| 73 | + | // waited on admission abandons records superseded by a completed purge. Holding | |
| 74 | + | // admission per record rather than per worker keeps the purge's drain latency | |
| 75 | + | // proportional to the slowest single record, not the whole catalogue. | |
| 76 | + | func (d *Daemon) restoreCatalogueEpoch(ctx context.Context, records []domain.CatalogueRecord, epoch uint64) { | |
| 53 | 77 | jobs := make(chan domain.CatalogueRecord) | |
| 54 | 78 | workers := min(catalogueRestoreConcurrency, len(records)) | |
| 55 | 79 | var wg sync.WaitGroup | |
| 56 | 80 | for range workers { | |
| 57 | 81 | wg.Go(func() { | |
| 58 | 82 | for record := range jobs { | |
| 59 | - | done := d.ensureCatalogueRegistryEntry(record) | |
| 60 | - | err := d.restoreRecord(ctx, record) | |
| 83 | + | if err := d.acquirePurgeAdmission(ctx); err != nil { | |
| 84 | + | // Admission is closed for good (shutdown or cancellation): drain | |
| 85 | + | // the queue so the producer can finish, and touch no registry state. | |
| 86 | + | continue | |
| 87 | + | } | |
| 88 | + | if ctx.Err() != nil { | |
| 89 | + | d.releasePurgeAdmission() | |
| 90 | + | continue | |
| 91 | + | } | |
| 92 | + | done, ok := d.ensureCatalogueRegistryEntry(record, epoch) | |
| 93 | + | if !ok { | |
| 94 | + | d.releasePurgeAdmission() | |
| 95 | + | continue | |
| 96 | + | } | |
| 97 | + | err := d.restoreRecord(ctx, record, epoch) | |
| 61 | 98 | d.finishRecordRestore(record, err, done) | |
| 99 | + | d.releasePurgeAdmission() | |
| 62 | 100 | } | |
| 63 | 101 | }) | |
| 64 | 102 | } | |
| @@ -100,15 +138,22 @@ func inactiveSessionFromRecord(record domain.CatalogueRecord, state protocol.Ses | |||
| 100 | 138 | } | |
| 101 | 139 | } | |
| 102 | 140 | ||
| 103 | - | func (d *Daemon) ensureCatalogueRegistryEntry(record domain.CatalogueRecord) chan struct{} { | |
| 141 | + | // ensureCatalogueRegistryEntry publishes a stopped registry entry for a record | |
| 142 | + | // captured at epoch, unless a purge or explicit shutdown has superseded it. The | |
| 143 | + | // false result means the record must be skipped: it no longer belongs to the | |
| 144 | + | // authoritative set, so creating an entry here would resurrect it. | |
| 145 | + | func (d *Daemon) ensureCatalogueRegistryEntry(record domain.CatalogueRecord, epoch uint64) (chan struct{}, bool) { | |
| 104 | 146 | d.mu.Lock() | |
| 105 | 147 | defer d.mu.Unlock() | |
| 148 | + | if d.purgeSupersededLocked(epoch) { | |
| 149 | + | return nil, false | |
| 150 | + | } | |
| 106 | 151 | if entry, ok := d.inactive[record.Name]; ok { | |
| 107 | - | return entry.restoreDone | |
| 152 | + | return entry.restoreDone, true | |
| 108 | 153 | } | |
| 109 | 154 | state, done := initialSessionState(record) | |
| 110 | 155 | d.inactive[record.Name] = inactiveSessionFromRecord(record, state, done) | |
| 111 | - | return done | |
| 156 | + | return done, true | |
| 112 | 157 | } | |
| 113 | 158 | ||
| 114 | 159 | // closeRuntimeRestoreDoneLocked closes a per-record restoration barrier. | |
| @@ -173,7 +218,12 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||
| 173 | 218 | } | |
| 174 | 219 | } | |
| 175 | 220 | ||
| 176 | - | func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecord) error { | |
| 221 | + | func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecord, epoch uint64) error { | |
| 222 | + | if d.purgeSuperseded(epoch) { | |
| 223 | + | // A purge or shutdown owns this record now; skip the load entirely so no | |
| 224 | + | // PTY is opened for a lifecycle that will not survive. | |
| 225 | + | return nil | |
| 226 | + | } | |
| 177 | 227 | if record.DegradedReason != "" { | |
| 178 | 228 | d.setStoppedRecovery(record, protocol.SessionBroken) | |
| 179 | 229 | d.logSessionDegraded(record, "persisted-broken") | |
| @@ -229,7 +279,7 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||
| 229 | 279 | d.log.Info("snapshot_checkpoint_reconciliation_complete", "session", record.Name, "incarnation", record.IncarnationID.String(), "generation", selected.Generation) | |
| 230 | 280 | } | |
| 231 | 281 | ||
| 232 | - | if err := d.restoreSession(ctx, selectedSnapshot, selectedGeneration.Generation, selected); err != nil { | |
| 282 | + | if err := d.restoreSession(ctx, selectedSnapshot, selectedGeneration.Generation, selected, epoch); err != nil { | |
| 233 | 283 | return err | |
| 234 | 284 | } | |
| 235 | 285 | d.setStoppedRecovery(record, protocol.SessionDown) | |
| @@ -39,26 +39,64 @@ func (d *Daemon) restoreIncrementalSnapshots(ctx context.Context) { | |||||
| 39 | if d.catalogue == nil { | 39 | if d.catalogue == nil { | ||
| 40 | return | 40 | return | ||
| 41 | } | 41 | } | ||
| 42 | + | records, epoch, ok := d.captureCatalogueSnapshot(ctx) | |||
| 43 | + | if !ok { | |||
| 44 | + | return | |||
| 45 | + | } | |||
| 46 | + | d.restoreCatalogueEpoch(ctx, records, epoch) | |||
| 47 | + | } | |||
| 48 | + | ||||
| 49 | + | // captureCatalogueSnapshot reads the purge boundary and the authoritative | |||
| 50 | + | // catalogue records together under purge admission. Holding the gate for both | |||
| 51 | + | // reads keeps them atomic with any KillAll: a purge can never increment the | |||
| 52 | + | // epoch between the two reads and pair a record set it already removed with a | |||
| 53 | + | // boundary that looks current. The caller's context and the move/purge gates | |||
| 54 | + | // bound the wait. | |||
| 55 | + | func (d *Daemon) captureCatalogueSnapshot(ctx context.Context) ([]domain.CatalogueRecord, uint64, bool) { | |||
| 56 | + | if err := d.acquirePurgeAdmission(ctx); err != nil { | |||
| 57 | + | return nil, 0, false | |||
| 58 | + | } | |||
| 59 | + | defer d.releasePurgeAdmission() | |||
| 60 | + | epoch := d.purgeEpochSnapshot() | |||
| 42 | records, err := d.catalogue.Records() | 61 | records, err := d.catalogue.Records() | ||
| 43 | if err != nil { | 62 | if err != nil { | ||
| 44 | d.log.Error("loading catalogue for snapshot restoration failed", "err", err) | 63 | d.log.Error("loading catalogue for snapshot restoration failed", "err", err) | ||
| 45 | - | return | 64 | + | return nil, 0, false |
| 46 | } | 65 | } | ||
| 47 | - | d.restoreCatalogue(ctx, records) | 66 | + | return records, epoch, true |
| 48 | } | 67 | } | ||
| 49 | 68 | ||||
| 50 | - | // restoreCatalogue gives every expected record an independent bounded job. | 69 | + | // restoreCatalogueEpoch restores records captured at the given purge boundary. |
| 51 | - | // A failed record is degraded without cancelling any sibling restoration. | 70 | + | // Each worker holds purge admission for exactly one record's transition — its |
| 52 | - | func (d *Daemon) restoreCatalogue(ctx context.Context, records []domain.CatalogueRecord) { | 71 | + | // registry publication, checkpoint load, and publication or demotion — so a |
| 72 | + | // KillAll drains the in-flight record before it snapshots, and a worker that | |||
| 73 | + | // waited on admission abandons records superseded by a completed purge. Holding | |||
| 74 | + | // admission per record rather than per worker keeps the purge's drain latency | |||
| 75 | + | // proportional to the slowest single record, not the whole catalogue. | |||
| 76 | + | func (d *Daemon) restoreCatalogueEpoch(ctx context.Context, records []domain.CatalogueRecord, epoch uint64) { | |||
| 53 | jobs := make(chan domain.CatalogueRecord) | 77 | jobs := make(chan domain.CatalogueRecord) | ||
| 54 | workers := min(catalogueRestoreConcurrency, len(records)) | 78 | workers := min(catalogueRestoreConcurrency, len(records)) | ||
| 55 | var wg sync.WaitGroup | 79 | var wg sync.WaitGroup | ||
| 56 | for range workers { | 80 | for range workers { | ||
| 57 | wg.Go(func() { | 81 | wg.Go(func() { | ||
| 58 | for record := range jobs { | 82 | for record := range jobs { | ||
| 59 | - | done := d.ensureCatalogueRegistryEntry(record) | 83 | + | if err := d.acquirePurgeAdmission(ctx); err != nil { |
| 60 | - | err := d.restoreRecord(ctx, record) | 84 | + | // Admission is closed for good (shutdown or cancellation): drain |
| 85 | + | // the queue so the producer can finish, and touch no registry state. | |||
| 86 | + | continue | |||
| 87 | + | } | |||
| 88 | + | if ctx.Err() != nil { | |||
| 89 | + | d.releasePurgeAdmission() | |||
| 90 | + | continue | |||
| 91 | + | } | |||
| 92 | + | done, ok := d.ensureCatalogueRegistryEntry(record, epoch) | |||
| 93 | + | if !ok { | |||
| 94 | + | d.releasePurgeAdmission() | |||
| 95 | + | continue | |||
| 96 | + | } | |||
| 97 | + | err := d.restoreRecord(ctx, record, epoch) | |||
| 61 | d.finishRecordRestore(record, err, done) | 98 | d.finishRecordRestore(record, err, done) | ||
| 99 | + | d.releasePurgeAdmission() | |||
| 62 | } | 100 | } | ||
| 63 | }) | 101 | }) | ||
| 64 | } | 102 | } | ||
| @@ -100,15 +138,22 @@ func inactiveSessionFromRecord(record domain.CatalogueRecord, state protocol.Ses | |||||
| 100 | } | 138 | } | ||
| 101 | } | 139 | } | ||
| 102 | 140 | ||||
| 103 | - | func (d *Daemon) ensureCatalogueRegistryEntry(record domain.CatalogueRecord) chan struct{} { | 141 | + | // ensureCatalogueRegistryEntry publishes a stopped registry entry for a record |
| 142 | + | // captured at epoch, unless a purge or explicit shutdown has superseded it. The | |||
| 143 | + | // false result means the record must be skipped: it no longer belongs to the | |||
| 144 | + | // authoritative set, so creating an entry here would resurrect it. | |||
| 145 | + | func (d *Daemon) ensureCatalogueRegistryEntry(record domain.CatalogueRecord, epoch uint64) (chan struct{}, bool) { | |||
| 104 | d.mu.Lock() | 146 | d.mu.Lock() | ||
| 105 | defer d.mu.Unlock() | 147 | defer d.mu.Unlock() | ||
| 148 | + | if d.purgeSupersededLocked(epoch) { | |||
| 149 | + | return nil, false | |||
| 150 | + | } | |||
| 106 | if entry, ok := d.inactive[record.Name]; ok { | 151 | if entry, ok := d.inactive[record.Name]; ok { | ||
| 107 | - | return entry.restoreDone | 152 | + | return entry.restoreDone, true |
| 108 | } | 153 | } | ||
| 109 | state, done := initialSessionState(record) | 154 | state, done := initialSessionState(record) | ||
| 110 | d.inactive[record.Name] = inactiveSessionFromRecord(record, state, done) | 155 | d.inactive[record.Name] = inactiveSessionFromRecord(record, state, done) | ||
| 111 | - | return done | 156 | + | return done, true |
| 112 | } | 157 | } | ||
| 113 | 158 | ||||
| 114 | // closeRuntimeRestoreDoneLocked closes a per-record restoration barrier. | 159 | // closeRuntimeRestoreDoneLocked closes a per-record restoration barrier. | ||
| @@ -173,7 +218,12 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||||
| 173 | } | 218 | } | ||
| 174 | } | 219 | } | ||
| 175 | 220 | ||||
| 176 | - | func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecord) error { | 221 | + | func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecord, epoch uint64) error { |
| 222 | + | if d.purgeSuperseded(epoch) { | |||
| 223 | + | // A purge or shutdown owns this record now; skip the load entirely so no | |||
| 224 | + | // PTY is opened for a lifecycle that will not survive. | |||
| 225 | + | return nil | |||
| 226 | + | } | |||
| 177 | if record.DegradedReason != "" { | 227 | if record.DegradedReason != "" { | ||
| 178 | d.setStoppedRecovery(record, protocol.SessionBroken) | 228 | d.setStoppedRecovery(record, protocol.SessionBroken) | ||
| 179 | d.logSessionDegraded(record, "persisted-broken") | 229 | d.logSessionDegraded(record, "persisted-broken") | ||
| @@ -229,7 +279,7 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||||
| 229 | d.log.Info("snapshot_checkpoint_reconciliation_complete", "session", record.Name, "incarnation", record.IncarnationID.String(), "generation", selected.Generation) | 279 | d.log.Info("snapshot_checkpoint_reconciliation_complete", "session", record.Name, "incarnation", record.IncarnationID.String(), "generation", selected.Generation) | ||
| 230 | } | 280 | } | ||
| 231 | 281 | ||||
| 232 | - | if err := d.restoreSession(ctx, selectedSnapshot, selectedGeneration.Generation, selected); err != nil { | 282 | + | if err := d.restoreSession(ctx, selectedSnapshot, selectedGeneration.Generation, selected, epoch); err != nil { |
| 233 | return err | 283 | return err | ||
| 234 | } | 284 | } | ||
| 235 | d.setStoppedRecovery(record, protocol.SessionDown) | 285 | d.setStoppedRecovery(record, protocol.SessionDown) | ||
M internal/usecase/daemon/snapshot_scheduler_test.go +92 -0
| @@ -83,6 +83,98 @@ func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | |||
| 83 | 83 | } | |
| 84 | 84 | } | |
| 85 | 85 | ||
| 86 | + | // TestQuarantineDiscardDoesNotDoubleDecrementPendingCaptures proves the pending | |
| 87 | + | // capture accounting is exact across a quarantine discard and a rollback | |
| 88 | + | // replacement: a queued capture the quarantine discarded already dropped its own | |
| 89 | + | // pending count, so the worker-side finish that rejects its stale identity must | |
| 90 | + | // not decrement a replacement scheduled after the rollback. | |
| 91 | + | func TestQuarantineDiscardDoesNotDoubleDecrementPendingCaptures(t *testing.T) { | |
| 92 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 93 | + | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 94 | + | WithSnapshotRepository(repository)(d) | |
| 95 | + | startSnapshotEncodeWorker(t, d) | |
| 96 | + | ||
| 97 | + | blockerStarted := make(chan struct{}) | |
| 98 | + | releaseBlocker := make(chan struct{}) | |
| 99 | + | var releaseOnce sync.Once | |
| 100 | + | release := func() { releaseOnce.Do(func() { close(releaseBlocker) }) } | |
| 101 | + | t.Cleanup(release) | |
| 102 | + | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | |
| 103 | + | if publication.Name == "blocker" { | |
| 104 | + | close(blockerStarted) | |
| 105 | + | <-releaseBlocker | |
| 106 | + | } | |
| 107 | + | return nil | |
| 108 | + | }).Maybe() | |
| 109 | + | ||
| 110 | + | // Occupy the single worker so the test session's capture stays queued. | |
| 111 | + | blocker := newSnapshotTestSession(t, "blocker", false, "/work") | |
| 112 | + | markSnapshotDirty(blocker) | |
| 113 | + | require.True(t, d.scheduleSnapshot(blocker)) | |
| 114 | + | awaitTestValue(t, blockerStarted, "the blocking publication did not start") | |
| 115 | + | ||
| 116 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 117 | + | markSnapshotDirty(sess) | |
| 118 | + | require.True(t, d.scheduleSnapshot(sess), "the test capture should queue behind the blocked worker") | |
| 119 | + | sess.snapshotMu.Lock() | |
| 120 | + | first := sess.snapshotQueuedCapture | |
| 121 | + | firstCount := sess.snapshotPendingCaptures | |
| 122 | + | sess.snapshotMu.Unlock() | |
| 123 | + | require.NotNil(t, first) | |
| 124 | + | require.Equal(t, uint(1), firstCount) | |
| 125 | + | ||
| 126 | + | // Quarantine discards the queued capture and drops its pending count exactly | |
| 127 | + | // once. | |
| 128 | + | <-quarantineSnapshotCoordinator(sess) | |
| 129 | + | sess.snapshotMu.Lock() | |
| 130 | + | discardedCount := sess.snapshotPendingCaptures | |
| 131 | + | sess.snapshotMu.Unlock() | |
| 132 | + | require.Equal(t, uint(0), discardedCount, "quarantine must consume the discarded capture's count") | |
| 133 | + | require.True(t, first.coordinatorDiscarded) | |
| 134 | + | ||
| 135 | + | // Model the worker having already dequeued the stale capture: the global | |
| 136 | + | // bounded queue still holds it, so free the slot before the replacement. | |
| 137 | + | select { | |
| 138 | + | case dequeued := <-d.snapshotJobs: | |
| 139 | + | require.Same(t, first, dequeued) | |
| 140 | + | default: | |
| 141 | + | t.Fatal("the discarded capture must still be queued in the global worker queue") | |
| 142 | + | } | |
| 143 | + | ||
| 144 | + | // The rollback restores scheduling, and the replacement capture owns its own | |
| 145 | + | // pending count. | |
| 146 | + | rollbackSnapshotQuarantine(sess, true) | |
| 147 | + | require.True(t, d.scheduleSnapshot(sess), "the rolled-back session must schedule a replacement capture") | |
| 148 | + | sess.snapshotMu.Lock() | |
| 149 | + | replacement := sess.snapshotQueuedCapture | |
| 150 | + | replacementCount := sess.snapshotPendingCaptures | |
| 151 | + | sess.snapshotMu.Unlock() | |
| 152 | + | require.NotNil(t, replacement) | |
| 153 | + | require.NotSame(t, first, replacement) | |
| 154 | + | require.Equal(t, uint(1), replacementCount) | |
| 155 | + | ||
| 156 | + | // The worker finishing the stale discarded capture must not decrement the | |
| 157 | + | // live replacement's pending count. | |
| 158 | + | d.finishSnapshotCapture(first, false) | |
| 159 | + | sess.snapshotMu.Lock() | |
| 160 | + | afterFinishCount := sess.snapshotPendingCaptures | |
| 161 | + | afterFinishPending := sess.snapshotPending | |
| 162 | + | sess.snapshotMu.Unlock() | |
| 163 | + | require.Equal(t, uint(1), afterFinishCount, "the discarded capture must not decrement the replacement") | |
| 164 | + | require.True(t, afterFinishPending) | |
| 165 | + | ||
| 166 | + | // Once the worker drains both captures only the replacement publishes, and | |
| 167 | + | // the accounting returns to zero. | |
| 168 | + | release() | |
| 169 | + | awaitSnapshotClean(t, sess) | |
| 170 | + | sess.snapshotMu.Lock() | |
| 171 | + | finalCount := sess.snapshotPendingCaptures | |
| 172 | + | finalPending := sess.snapshotPending | |
| 173 | + | sess.snapshotMu.Unlock() | |
| 174 | + | require.Equal(t, uint(0), finalCount) | |
| 175 | + | require.False(t, finalPending) | |
| 176 | + | } | |
| 177 | + | ||
| 86 | 178 | func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | |
| 87 | 179 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 88 | 180 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| @@ -83,6 +83,98 @@ func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | |||||
| 83 | } | 83 | } | ||
| 84 | } | 84 | } | ||
| 85 | 85 | ||||
| 86 | + | // TestQuarantineDiscardDoesNotDoubleDecrementPendingCaptures proves the pending | |||
| 87 | + | // capture accounting is exact across a quarantine discard and a rollback | |||
| 88 | + | // replacement: a queued capture the quarantine discarded already dropped its own | |||
| 89 | + | // pending count, so the worker-side finish that rejects its stale identity must | |||
| 90 | + | // not decrement a replacement scheduled after the rollback. | |||
| 91 | + | func TestQuarantineDiscardDoesNotDoubleDecrementPendingCaptures(t *testing.T) { | |||
| 92 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 93 | + | repository := portsmocks.NewMockSnapshotRepository(t) | |||
| 94 | + | WithSnapshotRepository(repository)(d) | |||
| 95 | + | startSnapshotEncodeWorker(t, d) | |||
| 96 | + | ||||
| 97 | + | blockerStarted := make(chan struct{}) | |||
| 98 | + | releaseBlocker := make(chan struct{}) | |||
| 99 | + | var releaseOnce sync.Once | |||
| 100 | + | release := func() { releaseOnce.Do(func() { close(releaseBlocker) }) } | |||
| 101 | + | t.Cleanup(release) | |||
| 102 | + | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | |||
| 103 | + | if publication.Name == "blocker" { | |||
| 104 | + | close(blockerStarted) | |||
| 105 | + | <-releaseBlocker | |||
| 106 | + | } | |||
| 107 | + | return nil | |||
| 108 | + | }).Maybe() | |||
| 109 | + | ||||
| 110 | + | // Occupy the single worker so the test session's capture stays queued. | |||
| 111 | + | blocker := newSnapshotTestSession(t, "blocker", false, "/work") | |||
| 112 | + | markSnapshotDirty(blocker) | |||
| 113 | + | require.True(t, d.scheduleSnapshot(blocker)) | |||
| 114 | + | awaitTestValue(t, blockerStarted, "the blocking publication did not start") | |||
| 115 | + | ||||
| 116 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 117 | + | markSnapshotDirty(sess) | |||
| 118 | + | require.True(t, d.scheduleSnapshot(sess), "the test capture should queue behind the blocked worker") | |||
| 119 | + | sess.snapshotMu.Lock() | |||
| 120 | + | first := sess.snapshotQueuedCapture | |||
| 121 | + | firstCount := sess.snapshotPendingCaptures | |||
| 122 | + | sess.snapshotMu.Unlock() | |||
| 123 | + | require.NotNil(t, first) | |||
| 124 | + | require.Equal(t, uint(1), firstCount) | |||
| 125 | + | ||||
| 126 | + | // Quarantine discards the queued capture and drops its pending count exactly | |||
| 127 | + | // once. | |||
| 128 | + | <-quarantineSnapshotCoordinator(sess) | |||
| 129 | + | sess.snapshotMu.Lock() | |||
| 130 | + | discardedCount := sess.snapshotPendingCaptures | |||
| 131 | + | sess.snapshotMu.Unlock() | |||
| 132 | + | require.Equal(t, uint(0), discardedCount, "quarantine must consume the discarded capture's count") | |||
| 133 | + | require.True(t, first.coordinatorDiscarded) | |||
| 134 | + | ||||
| 135 | + | // Model the worker having already dequeued the stale capture: the global | |||
| 136 | + | // bounded queue still holds it, so free the slot before the replacement. | |||
| 137 | + | select { | |||
| 138 | + | case dequeued := <-d.snapshotJobs: | |||
| 139 | + | require.Same(t, first, dequeued) | |||
| 140 | + | default: | |||
| 141 | + | t.Fatal("the discarded capture must still be queued in the global worker queue") | |||
| 142 | + | } | |||
| 143 | + | ||||
| 144 | + | // The rollback restores scheduling, and the replacement capture owns its own | |||
| 145 | + | // pending count. | |||
| 146 | + | rollbackSnapshotQuarantine(sess, true) | |||
| 147 | + | require.True(t, d.scheduleSnapshot(sess), "the rolled-back session must schedule a replacement capture") | |||
| 148 | + | sess.snapshotMu.Lock() | |||
| 149 | + | replacement := sess.snapshotQueuedCapture | |||
| 150 | + | replacementCount := sess.snapshotPendingCaptures | |||
| 151 | + | sess.snapshotMu.Unlock() | |||
| 152 | + | require.NotNil(t, replacement) | |||
| 153 | + | require.NotSame(t, first, replacement) | |||
| 154 | + | require.Equal(t, uint(1), replacementCount) | |||
| 155 | + | ||||
| 156 | + | // The worker finishing the stale discarded capture must not decrement the | |||
| 157 | + | // live replacement's pending count. | |||
| 158 | + | d.finishSnapshotCapture(first, false) | |||
| 159 | + | sess.snapshotMu.Lock() | |||
| 160 | + | afterFinishCount := sess.snapshotPendingCaptures | |||
| 161 | + | afterFinishPending := sess.snapshotPending | |||
| 162 | + | sess.snapshotMu.Unlock() | |||
| 163 | + | require.Equal(t, uint(1), afterFinishCount, "the discarded capture must not decrement the replacement") | |||
| 164 | + | require.True(t, afterFinishPending) | |||
| 165 | + | ||||
| 166 | + | // Once the worker drains both captures only the replacement publishes, and | |||
| 167 | + | // the accounting returns to zero. | |||
| 168 | + | release() | |||
| 169 | + | awaitSnapshotClean(t, sess) | |||
| 170 | + | sess.snapshotMu.Lock() | |||
| 171 | + | finalCount := sess.snapshotPendingCaptures | |||
| 172 | + | finalPending := sess.snapshotPending | |||
| 173 | + | sess.snapshotMu.Unlock() | |||
| 174 | + | require.Equal(t, uint(0), finalCount) | |||
| 175 | + | require.False(t, finalPending) | |||
| 176 | + | } | |||
| 177 | + | ||||
| 86 | func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | 178 | func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | ||
| 87 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 179 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 88 | repository := portsmocks.NewMockSnapshotRepository(t) | 180 | repository := portsmocks.NewMockSnapshotRepository(t) | ||