1//go:build linux
2
3package main
4
5import (
6 "errors"
7 "fmt"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "syscall"
12 "testing"
13 "time"
14)
15
16// TestClientTracePairsAfterPTYHangup drives the real teardown escalation: a
17// live client is severed by closing its controlling PTY master (the harness
18// hangup stage) with no graceful "exit\n". A client that dies from the default
19// SIGHUP disposition truncates its receive pump's in-flight span, so its trace
20// merge fails with an unpaired adapter_receive_start. A client that survives the
21// hangup closes its transport and serializes the matching receive-end first, so
22// the merge pairs.
23func TestClientTracePairsAfterPTYHangup(t *testing.T) {
24 if testing.Short() {
25 t.Skip("public CLI hangup lifecycle")
26 }
27 root, err := filepath.Abs(filepath.Join("..", ".."))
28 if err != nil {
29 t.Fatal(err)
30 }
31 // Keep the run directory (and thus the Unix socket path) short: MkdirTemp in
32 // the system temp root stays well under the platform socket-path limit.
33 runDir, err := os.MkdirTemp("", "vev-hangup-")
34 if err != nil {
35 t.Fatal(err)
36 }
37 removeTestTree(t, runDir)
38 bin := filepath.Join(runDir, "vev")
39 build := exec.Command("go", "build", "-o", bin, "./")
40 build.Dir = root
41 if output, err := build.CombinedOutput(); err != nil {
42 t.Fatalf("build public CLI: %v\n%s", err, output)
43 }
44
45 s := scenario{ID: "hangup-local", Transport: "local", Workload: "idle", Roles: []string{"daemon", "client"}}
46 selected, err := manifestTransport(manifest{}, s.Transport)
47 if err != nil {
48 t.Fatal(err)
49 }
50
51 maps := make([]processMapping, 0, len(s.Roles))
52 for i, role := range s.Roles {
53 pid := fmt.Sprintf("%s-r%03d-%s-%02d", safeName(s.ID), 1, safeName(role), i+1)
54 path := filepath.Join(runDir, pid+".jsonl")
55 f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
56 if err != nil {
57 t.Fatal(err)
58 }
59 if err := f.Close(); err != nil {
60 t.Fatal(err)
61 }
62 maps = append(maps, processMapping{ProcessID: pid, ClockDomain: pid, TracePath: path, Role: role, Scenario: s.ID, Run: 1, Identity: "harness-assigned:" + pid})
63 }
64
65 launcher := &cliLauncher{bin: bin}
66 var daemonProc launchedProcess
67 var client *cliProcess
68 var clientMapping processMapping
69 for _, pm := range launchOrder(maps) {
70 p, err := launcher.Launch(pm, routeRoleArgs(s, pm, selected))
71 if err != nil {
72 t.Fatalf("launch %s: %v", pm.Role, err)
73 }
74 switch pm.Role {
75 case "daemon":
76 daemonProc = p
77 ready, ok := p.(interface{ WaitReady() error })
78 if !ok {
79 t.Fatal("daemon role does not expose WaitReady")
80 }
81 if err := ready.WaitReady(); err != nil {
82 t.Fatalf("daemon readiness: %v", err)
83 }
84 case "client":
85 cp, ok := p.(*cliProcess)
86 if !ok {
87 t.Fatalf("client role is %T, want *cliProcess", p)
88 }
89 client, clientMapping = cp, pm
90 }
91 }
92 if client == nil || daemonProc == nil {
93 t.Fatal("did not launch both roles")
94 }
95 // Close the daemon last, on every exit path, so a failed assertion cannot
96 // leak a foreground daemon or its runtime directory.
97 defer func() {
98 if err := daemonProc.Close(); err != nil {
99 t.Errorf("close daemon role: %v", err)
100 }
101 }()
102 // Close the client on every exit path so a failed assertion cannot leak the
103 // process. Close is idempotent (sync.Once): the explicit call below, once
104 // reached, is the one whose result is asserted; this defer is a backstop.
105 defer func() {
106 if err := client.Close(); err != nil {
107 t.Errorf("close client role: %v", err)
108 }
109 }()
110
111 // Wait until the broker-backed client reaches a real attached shell before
112 // severing its terminal. Session carriage tracing now belongs to the broker,
113 // not the thin client process.
114 if err := client.waitForTerminalReady(); err != nil {
115 t.Fatalf("client never reached an attached shell: %v", err)
116 }
117
118 // Sever the controlling terminal with no graceful exit. This is the harness
119 // hangup escalation stage, delivering SIGHUP to the client session leader.
120 if err := client.pty.Close(); err != nil {
121 t.Fatalf("close client PTY master: %v", err)
122 }
123 // Close runs the bounded escalation ladder; the hangup above already severed
124 // the PTY, so this should reap gracefully without reaching SIGTERM/SIGKILL.
125 if err := client.Close(); err != nil {
126 t.Fatalf("client did not exit cleanly after PTY hangup: %v", err)
127 }
128
129 // The severed client's trace must pair. On the unfixed client this fails with
130 // "missing process-local span pair" for the truncated adapter_receive span.
131 if _, err := mergeProcessTraces([]processMapping{clientMapping}); err != nil {
132 t.Fatalf("client trace did not pair after PTY hangup: %v", err)
133 }
134}
135
136// TestClientGracefulShutdownDrainsTracedStdioDescendant proves the launcher's
137// normal client stop exits the attached shell before sending any signal. The
138// client then closes and waits for its ssh transport, giving the traced broker mux stdio
139// descendant time to serialize its blocked receive's end mark and exit before
140// the client is reaped.
141func TestClientGracefulShutdownDrainsTracedStdioDescendant(t *testing.T) {
142 if testing.Short() {
143 t.Skip("public CLI stdio descendant lifecycle")
144 }
145 root, err := filepath.Abs(filepath.Join("..", ".."))
146 if err != nil {
147 t.Fatal(err)
148 }
149 runDir, err := os.MkdirTemp("", "vev-stdio-drain-")
150 if err != nil {
151 t.Fatal(err)
152 }
153 removeTestTree(t, runDir)
154 bin := filepath.Join(runDir, "vev")
155 build := exec.Command("go", "build", "-o", bin, "./")
156 build.Dir = root
157 if output, err := build.CombinedOutput(); err != nil {
158 t.Fatalf("build public CLI: %v\n%s", err, output)
159 }
160
161 s := scenario{ID: "stdio-drain", Roles: []string{"daemon", "ssh_stdio_peer", "client"}}
162 maps := make([]processMapping, 0, len(s.Roles))
163 for i, role := range s.Roles {
164 id := fmt.Sprintf("%s-r001-%s-%02d", safeName(s.ID), safeName(role), i+1)
165 path := filepath.Join(runDir, id+".jsonl")
166 f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
167 if err != nil {
168 t.Fatal(err)
169 }
170 if err := f.Close(); err != nil {
171 t.Fatal(err)
172 }
173 maps = append(maps, processMapping{ProcessID: id, ClockDomain: id, TracePath: path, Role: role, Scenario: s.ID, Run: 1, Identity: "harness-assigned:" + id})
174 }
175
176 launcher := &cliLauncher{bin: bin}
177 var daemon, peer launchedProcess
178 var client *cliProcess
179 var peerMapping processMapping
180 for _, pm := range launchOrder(maps) {
181 var command roleCommand
182 switch pm.Role {
183 case "daemon":
184 command = routeRoleArgs(s, pm, transport{})
185 case "ssh_stdio_peer":
186 // The broker mux stdio helper carries the remote daemon connection.
187 command = roleCommand{Args: []string{"_broker-mux-stdio", "--production"}, Transport: transport{ID: "ssh_stdio", Kind: "ssh_stdio"}}
188 case "client":
189 command = roleCommand{Args: []string{"attach", "harness@127.0.0.1"}}
190 }
191 process, err := launcher.Launch(pm, command)
192 if err != nil {
193 t.Fatalf("launch %s: %v", pm.Role, err)
194 }
195 switch pm.Role {
196 case "daemon":
197 daemon = process
198 readyProcess, ok := process.(interface{ WaitReady() error })
199 if !ok {
200 t.Fatalf("daemon process type %T does not support readiness", process)
201 }
202 if err := readyProcess.WaitReady(); err != nil {
203 t.Fatalf("daemon readiness: %v", err)
204 }
205 case "ssh_stdio_peer":
206 peer, peerMapping = process, pm
207 case "client":
208 var ok bool
209 client, ok = process.(*cliProcess)
210 if !ok {
211 t.Fatalf("client process type = %T, want *cliProcess", process)
212 }
213 }
214 }
215 if daemon == nil || peer == nil || client == nil {
216 t.Fatal("did not launch complete stdio process tree")
217 }
218 defer func() {
219 if err := peer.Close(); err != nil {
220 t.Errorf("close peer role: %v", err)
221 }
222 if err := daemon.Close(); err != nil {
223 t.Errorf("close daemon role: %v", err)
224 }
225 }()
226 defer func() {
227 if err := client.Close(); err != nil {
228 t.Errorf("close client role: %v", err)
229 }
230 }()
231
232 if err := client.waitForTerminalReady(); err != nil {
233 t.Fatalf("stdio client never reached an attached shell: %v", err)
234 }
235 processGroup := client.cmd.Process.Pid
236 if err := client.Close(); err != nil {
237 t.Fatalf("graceful client close: %v", err)
238 }
239 if _, err := mergeProcessTraces([]processMapping{peerMapping}); err != nil {
240 t.Fatalf("stdio descendant trace did not drain: %v", err)
241 }
242 if err := waitForProcessGroupGone(processGroup, 10*time.Second); err != nil {
243 t.Fatal(err)
244 }
245}
246
247func waitForProcessGroupGone(pgid int, timeout time.Duration) error {
248 deadline := time.Now().Add(timeout)
249 for {
250 err := syscall.Kill(-pgid, 0)
251 if errors.Is(err, syscall.ESRCH) {
252 return nil
253 }
254 if err != nil {
255 return fmt.Errorf("probe process group %d: %w", pgid, err)
256 }
257 if time.Now().After(deadline) {
258 return fmt.Errorf("process group %d remained after client shutdown", pgid)
259 }
260 time.Sleep(10 * time.Millisecond)
261 }
262}