1//go:build linux
2
3package main
4
5import (
6 "encoding/json"
7 "errors"
8 "fmt"
9 "os"
10 "os/exec"
11 "path/filepath"
12 "strings"
13 "syscall"
14 "testing"
15 "time"
16)
17
18// TestClientTracePairsAfterPTYHangup drives the real teardown escalation: a
19// live client is severed by closing its controlling PTY master (the harness
20// hangup stage) with no graceful "exit\n". A client that dies from the default
21// SIGHUP disposition truncates its receive pump's in-flight span, so its trace
22// merge fails with an unpaired adapter_receive_start. A client that survives the
23// hangup closes its transport and serializes the matching receive-end first, so
24// the merge pairs.
25func TestClientTracePairsAfterPTYHangup(t *testing.T) {
26 if testing.Short() {
27 t.Skip("public CLI hangup lifecycle")
28 }
29 root, err := filepath.Abs(filepath.Join("..", ".."))
30 if err != nil {
31 t.Fatal(err)
32 }
33 // Keep the run directory (and thus the Unix socket path) short: MkdirTemp in
34 // the system temp root stays well under the platform socket-path limit.
35 runDir, err := os.MkdirTemp("", "vev-hangup-")
36 if err != nil {
37 t.Fatal(err)
38 }
39 removeTestTree(t, runDir)
40 bin := filepath.Join(runDir, "vev")
41 build := exec.Command("go", "build", "-o", bin, "./")
42 build.Dir = root
43 if output, err := build.CombinedOutput(); err != nil {
44 t.Fatalf("build public CLI: %v\n%s", err, output)
45 }
46
47 s := scenario{ID: "hangup-local", Transport: "local", Workload: "idle", Roles: []string{"daemon", "client"}}
48 selected, err := manifestTransport(manifest{}, s.Transport)
49 if err != nil {
50 t.Fatal(err)
51 }
52
53 maps := make([]processMapping, 0, len(s.Roles))
54 for i, role := range s.Roles {
55 pid := fmt.Sprintf("%s-r%03d-%s-%02d", safeName(s.ID), 1, safeName(role), i+1)
56 path := filepath.Join(runDir, pid+".jsonl")
57 f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
58 if err != nil {
59 t.Fatal(err)
60 }
61 if err := f.Close(); err != nil {
62 t.Fatal(err)
63 }
64 maps = append(maps, processMapping{ProcessID: pid, ClockDomain: pid, TracePath: path, Role: role, Scenario: s.ID, Run: 1, Identity: "harness-assigned:" + pid})
65 }
66
67 launcher := &cliLauncher{bin: bin}
68 var daemonProc launchedProcess
69 var client *cliProcess
70 var clientMapping processMapping
71 for _, pm := range launchOrder(maps) {
72 p, err := launcher.Launch(pm, routeRoleArgs(s, pm, selected))
73 if err != nil {
74 t.Fatalf("launch %s: %v", pm.Role, err)
75 }
76 switch pm.Role {
77 case "daemon":
78 daemonProc = p
79 ready, ok := p.(interface{ WaitReady() error })
80 if !ok {
81 t.Fatal("daemon role does not expose WaitReady")
82 }
83 if err := ready.WaitReady(); err != nil {
84 t.Fatalf("daemon readiness: %v", err)
85 }
86 case "client":
87 cp, ok := p.(*cliProcess)
88 if !ok {
89 t.Fatalf("client role is %T, want *cliProcess", p)
90 }
91 client, clientMapping = cp, pm
92 }
93 }
94 if client == nil || daemonProc == nil {
95 t.Fatal("did not launch both roles")
96 }
97 // Close the daemon last, on every exit path, so a failed assertion cannot
98 // leak a foreground daemon or its runtime directory.
99 defer func() {
100 if err := daemonProc.Close(); err != nil {
101 t.Errorf("close daemon role: %v", err)
102 }
103 }()
104 // Close the client on every exit path so a failed assertion cannot leak the
105 // process. Close is idempotent (sync.Once): the explicit call below, once
106 // reached, is the one whose result is asserted; this defer is a backstop.
107 defer func() {
108 if err := client.Close(); err != nil {
109 t.Errorf("close client role: %v", err)
110 }
111 }()
112
113 // Wait until the client is attached with a receive pump blocked mid-frame:
114 // its trace then holds one more adapter_receive_start than end. That in-flight
115 // span is exactly what an ungraceful death truncates.
116 if err := waitForInflightReceive(clientMapping.TracePath, 10*time.Second); err != nil {
117 t.Fatalf("client never reached a blocked receive: %v", err)
118 }
119
120 // Sever the controlling terminal with no graceful exit. This is the harness
121 // hangup escalation stage, delivering SIGHUP to the client session leader.
122 if err := client.pty.Close(); err != nil {
123 t.Fatalf("close client PTY master: %v", err)
124 }
125 // Close runs the bounded escalation ladder; the hangup above already severed
126 // the PTY, so this should reap gracefully without reaching SIGTERM/SIGKILL.
127 if err := client.Close(); err != nil {
128 t.Fatalf("client did not exit cleanly after PTY hangup: %v", err)
129 }
130
131 // The severed client's trace must pair. On the unfixed client this fails with
132 // "missing process-local span pair" for the truncated adapter_receive span.
133 if _, err := mergeProcessTraces([]processMapping{clientMapping}); err != nil {
134 t.Fatalf("client trace did not pair after PTY hangup: %v", err)
135 }
136}
137
138// TestClientGracefulShutdownDrainsTracedStdioDescendant proves the launcher's
139// normal client stop exits the attached shell before sending any signal. The
140// client then closes and waits for its ssh transport, giving the traced _stdio
141// descendant time to serialize its blocked receive's end mark and exit before
142// the client is reaped.
143func TestClientGracefulShutdownDrainsTracedStdioDescendant(t *testing.T) {
144 if testing.Short() {
145 t.Skip("public CLI stdio descendant lifecycle")
146 }
147 root, err := filepath.Abs(filepath.Join("..", ".."))
148 if err != nil {
149 t.Fatal(err)
150 }
151 runDir, err := os.MkdirTemp("", "vev-stdio-drain-")
152 if err != nil {
153 t.Fatal(err)
154 }
155 removeTestTree(t, runDir)
156 bin := filepath.Join(runDir, "vev")
157 build := exec.Command("go", "build", "-o", bin, "./")
158 build.Dir = root
159 if output, err := build.CombinedOutput(); err != nil {
160 t.Fatalf("build public CLI: %v\n%s", err, output)
161 }
162
163 s := scenario{ID: "stdio-drain", Roles: []string{"daemon", "ssh_stdio_peer", "client"}}
164 maps := make([]processMapping, 0, len(s.Roles))
165 for i, role := range s.Roles {
166 id := fmt.Sprintf("%s-r001-%s-%02d", safeName(s.ID), safeName(role), i+1)
167 path := filepath.Join(runDir, id+".jsonl")
168 f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
169 if err != nil {
170 t.Fatal(err)
171 }
172 if err := f.Close(); err != nil {
173 t.Fatal(err)
174 }
175 maps = append(maps, processMapping{ProcessID: id, ClockDomain: id, TracePath: path, Role: role, Scenario: s.ID, Run: 1, Identity: "harness-assigned:" + id})
176 }
177
178 launcher := &cliLauncher{bin: bin}
179 var daemon, peer launchedProcess
180 var client *cliProcess
181 var peerMapping processMapping
182 for _, pm := range launchOrder(maps) {
183 var command roleCommand
184 switch pm.Role {
185 case "daemon":
186 command = roleCommand{Args: []string{"--daemon"}}
187 case "ssh_stdio_peer":
188 // The transport-neutral _stdio mode requests an ephemeral session.
189 command = roleCommand{Args: []string{"_stdio"}, Transport: transport{ID: "ssh_stdio", Kind: "ssh_stdio"}}
190 case "client":
191 command = roleCommand{Args: []string{"attach", "harness@127.0.0.1"}}
192 }
193 process, err := launcher.Launch(pm, command)
194 if err != nil {
195 t.Fatalf("launch %s: %v", pm.Role, err)
196 }
197 switch pm.Role {
198 case "daemon":
199 daemon = process
200 readyProcess, ok := process.(interface{ WaitReady() error })
201 if !ok {
202 t.Fatalf("daemon process type %T does not support readiness", process)
203 }
204 if err := readyProcess.WaitReady(); err != nil {
205 t.Fatalf("daemon readiness: %v", err)
206 }
207 case "ssh_stdio_peer":
208 peer, peerMapping = process, pm
209 case "client":
210 var ok bool
211 client, ok = process.(*cliProcess)
212 if !ok {
213 t.Fatalf("client process type = %T, want *cliProcess", process)
214 }
215 }
216 }
217 if daemon == nil || peer == nil || client == nil {
218 t.Fatal("did not launch complete stdio process tree")
219 }
220 defer func() {
221 if err := peer.Close(); err != nil {
222 t.Errorf("close peer role: %v", err)
223 }
224 if err := daemon.Close(); err != nil {
225 t.Errorf("close daemon role: %v", err)
226 }
227 }()
228 defer func() {
229 if err := client.Close(); err != nil {
230 t.Errorf("close client role: %v", err)
231 }
232 }()
233
234 if err := waitForInflightReceive(peerMapping.TracePath, 10*time.Second); err != nil {
235 t.Fatalf("stdio descendant never reached blocked receive: %v", err)
236 }
237 processGroup := client.cmd.Process.Pid
238 if err := client.Close(); err != nil {
239 t.Fatalf("graceful client close: %v", err)
240 }
241 if _, err := mergeProcessTraces([]processMapping{peerMapping}); err != nil {
242 t.Fatalf("stdio descendant trace did not drain: %v", err)
243 }
244 if err := waitForProcessGroupGone(processGroup, 10*time.Second); err != nil {
245 t.Fatal(err)
246 }
247}
248
249func waitForProcessGroupGone(pgid int, timeout time.Duration) error {
250 deadline := time.Now().Add(timeout)
251 for {
252 err := syscall.Kill(-pgid, 0)
253 if errors.Is(err, syscall.ESRCH) {
254 return nil
255 }
256 if err != nil {
257 return fmt.Errorf("probe process group %d: %w", pgid, err)
258 }
259 if time.Now().After(deadline) {
260 return fmt.Errorf("process group %d remained after client shutdown", pgid)
261 }
262 time.Sleep(10 * time.Millisecond)
263 }
264}
265
266// waitForInflightReceive polls a process trace until it holds an unpaired
267// adapter_receive_start (more starts than ends). It tolerates a partially
268// written trailing line while the traced process appends concurrently.
269func waitForInflightReceive(path string, timeout time.Duration) error {
270 deadline := time.Now().Add(timeout)
271 for {
272 starts, ends := 0, 0
273 data, err := os.ReadFile(path)
274 if err != nil {
275 return err
276 }
277 for _, line := range strings.Split(string(data), "\n") {
278 line = strings.TrimSpace(line)
279 if line == "" {
280 continue
281 }
282 var r traceRecord
283 if json.Unmarshal([]byte(line), &r) != nil {
284 continue // a concurrently appended, not-yet-complete final line
285 }
286 switch r.Kind {
287 case "adapter_receive_start":
288 starts++
289 case "adapter_receive_end":
290 ends++
291 }
292 }
293 if starts > ends {
294 return nil
295 }
296 if time.Now().After(deadline) {
297 return fmt.Errorf("adapter_receive starts=%d ends=%d, want an in-flight start", starts, ends)
298 }
299 time.Sleep(10 * time.Millisecond)
300 }
301}