1//go:build linux
2
3package main
4
5import (
6 "encoding/json"
7 "errors"
8 "fmt"
9 "os"
10 "os/exec"
11 "path/filepath"
12 "strings"
13 "syscall"
14 "testing"
15 "time"
16)
17
18// TestClientTracePairsAfterPTYHangup drives the real teardown escalation: a
19// live client is severed by closing its controlling PTY master (the harness
20// hangup stage) with no graceful "exit\n". A client that dies from the default
21// SIGHUP disposition truncates its receive pump's in-flight span, so its trace
22// merge fails with an unpaired adapter_receive_start. A client that survives the
23// hangup closes its transport and serializes the matching receive-end first, so
24// the merge pairs.
25func TestClientTracePairsAfterPTYHangup(t *testing.T) {
26 if testing.Short() {
27 t.Skip("public CLI hangup lifecycle")
28 }
29 root, err := filepath.Abs(filepath.Join("..", ".."))
30 if err != nil {
31 t.Fatal(err)
32 }
33 // Keep the run directory (and thus the Unix socket path) short: MkdirTemp in
34 // the system temp root stays well under the platform socket-path limit.
35 runDir, err := os.MkdirTemp("", "vev-hangup-")
36 if err != nil {
37 t.Fatal(err)
38 }
39 removeTestTree(t, runDir)
40 bin := filepath.Join(runDir, "vev")
41 build := exec.Command("go", "build", "-o", bin, "./")
42 build.Dir = root
43 if output, err := build.CombinedOutput(); err != nil {
44 t.Fatalf("build public CLI: %v\n%s", err, output)
45 }
46
47 s := scenario{ID: "hangup-local", Transport: "local", Workload: "idle", Roles: []string{"daemon", "client"}}
48 selected, err := manifestTransport(manifest{}, s.Transport)
49 if err != nil {
50 t.Fatal(err)
51 }
52
53 maps := make([]processMapping, 0, len(s.Roles))
54 for i, role := range s.Roles {
55 pid := fmt.Sprintf("%s-r%03d-%s-%02d", safeName(s.ID), 1, safeName(role), i+1)
56 path := filepath.Join(runDir, pid+".jsonl")
57 f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
58 if err != nil {
59 t.Fatal(err)
60 }
61 if err := f.Close(); err != nil {
62 t.Fatal(err)
63 }
64 maps = append(maps, processMapping{ProcessID: pid, ClockDomain: pid, TracePath: path, Role: role, Scenario: s.ID, Run: 1, Identity: "harness-assigned:" + pid})
65 }
66
67 launcher := &cliLauncher{bin: bin}
68 var daemonProc launchedProcess
69 var client *cliProcess
70 var clientMapping processMapping
71 for _, pm := range launchOrder(maps) {
72 p, err := launcher.Launch(pm, routeRoleArgs(s, pm, selected))
73 if err != nil {
74 t.Fatalf("launch %s: %v", pm.Role, err)
75 }
76 switch pm.Role {
77 case "daemon":
78 daemonProc = p
79 ready, ok := p.(interface{ WaitReady() error })
80 if !ok {
81 t.Fatal("daemon role does not expose WaitReady")
82 }
83 if err := ready.WaitReady(); err != nil {
84 t.Fatalf("daemon readiness: %v", err)
85 }
86 case "client":
87 cp, ok := p.(*cliProcess)
88 if !ok {
89 t.Fatalf("client role is %T, want *cliProcess", p)
90 }
91 client, clientMapping = cp, pm
92 }
93 }
94 if client == nil || daemonProc == nil {
95 t.Fatal("did not launch both roles")
96 }
97 // Close the daemon last, on every exit path, so a failed assertion cannot
98 // leak a foreground daemon or its runtime directory.
99 defer func() {
100 if err := daemonProc.Close(); err != nil {
101 t.Errorf("close daemon role: %v", err)
102 }
103 }()
104 // Close the client on every exit path so a failed assertion cannot leak the
105 // process. Close is idempotent (sync.Once): the explicit call below, once
106 // reached, is the one whose result is asserted; this defer is a backstop.
107 defer func() {
108 if err := client.Close(); err != nil {
109 t.Errorf("close client role: %v", err)
110 }
111 }()
112
113 // Wait until the client is attached with a receive pump blocked mid-frame:
114 // its trace then holds one more adapter_receive_start than end. That in-flight
115 // span is exactly what an ungraceful death truncates.
116 if err := waitForInflightReceive(clientMapping.TracePath, 10*time.Second); err != nil {
117 t.Fatalf("client never reached a blocked receive: %v", err)
118 }
119
120 // Sever the controlling terminal with no graceful exit. This is the harness
121 // hangup escalation stage, delivering SIGHUP to the client session leader.
122 if err := client.pty.Close(); err != nil {
123 t.Fatalf("close client PTY master: %v", err)
124 }
125 // Close runs the bounded escalation ladder; the hangup above already severed
126 // the PTY, so this should reap gracefully without reaching SIGTERM/SIGKILL.
127 if err := client.Close(); err != nil {
128 t.Fatalf("client did not exit cleanly after PTY hangup: %v", err)
129 }
130
131 // The severed client's trace must pair. On the unfixed client this fails with
132 // "missing process-local span pair" for the truncated adapter_receive span.
133 if _, err := mergeProcessTraces([]processMapping{clientMapping}); err != nil {
134 t.Fatalf("client trace did not pair after PTY hangup: %v", err)
135 }
136}
137
138// TestClientGracefulShutdownDrainsTracedStdioDescendant proves the launcher's
139// normal client stop exits the attached shell before sending any signal. The
140// client then closes and waits for its ssh transport, giving the traced _stdio
141// descendant time to serialize its blocked receive's end mark and exit before
142// the client is reaped.
143func TestClientGracefulShutdownDrainsTracedStdioDescendant(t *testing.T) {
144 if testing.Short() {
145 t.Skip("public CLI stdio descendant lifecycle")
146 }
147 root, err := filepath.Abs(filepath.Join("..", ".."))
148 if err != nil {
149 t.Fatal(err)
150 }
151 runDir, err := os.MkdirTemp("", "vev-stdio-drain-")
152 if err != nil {
153 t.Fatal(err)
154 }
155 removeTestTree(t, runDir)
156 bin := filepath.Join(runDir, "vev")
157 build := exec.Command("go", "build", "-o", bin, "./")
158 build.Dir = root
159 if output, err := build.CombinedOutput(); err != nil {
160 t.Fatalf("build public CLI: %v\n%s", err, output)
161 }
162
163 s := scenario{ID: "stdio-drain", Roles: []string{"daemon", "ssh_stdio_peer", "client"}}
164 maps := make([]processMapping, 0, len(s.Roles))
165 for i, role := range s.Roles {
166 id := fmt.Sprintf("%s-r001-%s-%02d", safeName(s.ID), safeName(role), i+1)
167 path := filepath.Join(runDir, id+".jsonl")
168 f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
169 if err != nil {
170 t.Fatal(err)
171 }
172 if err := f.Close(); err != nil {
173 t.Fatal(err)
174 }
175 maps = append(maps, processMapping{ProcessID: id, ClockDomain: id, TracePath: path, Role: role, Scenario: s.ID, Run: 1, Identity: "harness-assigned:" + id})
176 }
177
178 launcher := &cliLauncher{bin: bin}
179 var daemon, peer launchedProcess
180 var client *cliProcess
181 var peerMapping processMapping
182 for _, pm := range launchOrder(maps) {
183 var command roleCommand
184 switch pm.Role {
185 case "daemon":
186 command = roleCommand{Args: []string{"--daemon"}}
187 case "ssh_stdio_peer":
188 // An empty remote session requests an ephemeral session, avoiding a
189 // separate bootstrap client while still executing the traced _stdio mode.
190 command = roleCommand{Args: []string{"_stdio", ""}, Transport: transport{ID: "ssh_stdio", Kind: "ssh_stdio"}}
191 case "client":
192 command = roleCommand{Args: []string{"attach", "harness@127.0.0.1"}}
193 }
194 process, err := launcher.Launch(pm, command)
195 if err != nil {
196 t.Fatalf("launch %s: %v", pm.Role, err)
197 }
198 switch pm.Role {
199 case "daemon":
200 daemon = process
201 readyProcess, ok := process.(interface{ WaitReady() error })
202 if !ok {
203 t.Fatalf("daemon process type %T does not support readiness", process)
204 }
205 if err := readyProcess.WaitReady(); err != nil {
206 t.Fatalf("daemon readiness: %v", err)
207 }
208 case "ssh_stdio_peer":
209 peer, peerMapping = process, pm
210 case "client":
211 var ok bool
212 client, ok = process.(*cliProcess)
213 if !ok {
214 t.Fatalf("client process type = %T, want *cliProcess", process)
215 }
216 }
217 }
218 if daemon == nil || peer == nil || client == nil {
219 t.Fatal("did not launch complete stdio process tree")
220 }
221 defer func() {
222 if err := peer.Close(); err != nil {
223 t.Errorf("close peer role: %v", err)
224 }
225 if err := daemon.Close(); err != nil {
226 t.Errorf("close daemon role: %v", err)
227 }
228 }()
229 defer func() {
230 if err := client.Close(); err != nil {
231 t.Errorf("close client role: %v", err)
232 }
233 }()
234
235 if err := waitForInflightReceive(peerMapping.TracePath, 10*time.Second); err != nil {
236 t.Fatalf("stdio descendant never reached blocked receive: %v", err)
237 }
238 processGroup := client.cmd.Process.Pid
239 if err := client.Close(); err != nil {
240 t.Fatalf("graceful client close: %v", err)
241 }
242 if _, err := mergeProcessTraces([]processMapping{peerMapping}); err != nil {
243 t.Fatalf("stdio descendant trace did not drain: %v", err)
244 }
245 if err := waitForProcessGroupGone(processGroup, 10*time.Second); err != nil {
246 t.Fatal(err)
247 }
248}
249
250func waitForProcessGroupGone(pgid int, timeout time.Duration) error {
251 deadline := time.Now().Add(timeout)
252 for {
253 err := syscall.Kill(-pgid, 0)
254 if errors.Is(err, syscall.ESRCH) {
255 return nil
256 }
257 if err != nil {
258 return fmt.Errorf("probe process group %d: %w", pgid, err)
259 }
260 if time.Now().After(deadline) {
261 return fmt.Errorf("process group %d remained after client shutdown", pgid)
262 }
263 time.Sleep(10 * time.Millisecond)
264 }
265}
266
267// waitForInflightReceive polls a process trace until it holds an unpaired
268// adapter_receive_start (more starts than ends). It tolerates a partially
269// written trailing line while the traced process appends concurrently.
270func waitForInflightReceive(path string, timeout time.Duration) error {
271 deadline := time.Now().Add(timeout)
272 for {
273 starts, ends := 0, 0
274 data, err := os.ReadFile(path)
275 if err != nil {
276 return err
277 }
278 for _, line := range strings.Split(string(data), "\n") {
279 line = strings.TrimSpace(line)
280 if line == "" {
281 continue
282 }
283 var r traceRecord
284 if json.Unmarshal([]byte(line), &r) != nil {
285 continue // a concurrently appended, not-yet-complete final line
286 }
287 switch r.Kind {
288 case "adapter_receive_start":
289 starts++
290 case "adapter_receive_end":
291 ends++
292 }
293 }
294 if starts > ends {
295 return nil
296 }
297 if time.Now().After(deadline) {
298 return fmt.Errorf("adapter_receive starts=%d ends=%d, want an in-flight start", starts, ends)
299 }
300 time.Sleep(10 * time.Millisecond)
301 }
302}