1package tools
2
3import (
4 "bytes"
5 "context"
6 "encoding/json"
7 "errors"
8 "os"
9 "strings"
10 "testing"
11 "time"
12
13 "github.com/bnema/zut/packages/provider"
14)
15
16func pythonArgsJSON(t *testing.T, code string, timeout int64) json.RawMessage {
17 t.Helper()
18 return mustJSON(t, map[string]any{"code": code, "timeout": timeout})
19}
20
21func stubPythonTool(resolved resolvedPython, resolveErr error, outcome pythonExecOutcome, execErr error) *PythonTool {
22 return &PythonTool{
23 CWD: "/session",
24 resolveHook: func(context.Context, string, string) (resolvedPython, error) {
25 return resolved, resolveErr
26 },
27 execHook: func(context.Context, string, string, string, func(string)) (pythonExecOutcome, error) {
28 return outcome, execErr
29 },
30 }
31}
32
33func TestPythonSchemaRequiresCodeAndTimeout(t *testing.T) {
34 var schema struct {
35 Properties map[string]struct {
36 Type string `json:"type"`
37 Minimum *int64 `json:"minimum"`
38 Maximum *int64 `json:"maximum"`
39 Description string `json:"description"`
40 } `json:"properties"`
41 Required []string `json:"required"`
42 }
43 if err := json.Unmarshal((&PythonTool{}).Schema(), &schema); err != nil {
44 t.Fatal(err)
45 }
46 timeout, ok := schema.Properties["timeout"]
47 if !ok || timeout.Type != "integer" || timeout.Minimum == nil || *timeout.Minimum != 1 || timeout.Maximum == nil || *timeout.Maximum != maxPythonTimeoutSeconds {
48 t.Fatalf("timeout schema = %+v", timeout)
49 }
50 if _, ok := schema.Properties["code"]; !ok {
51 t.Fatal("code property missing")
52 }
53 required := map[string]bool{}
54 for _, name := range schema.Required {
55 required[name] = true
56 }
57 if !required["code"] || !required["timeout"] {
58 t.Fatalf("required = %v", schema.Required)
59 }
60 if (&PythonTool{}).Name() != "python" {
61 t.Fatalf("name = %q", (&PythonTool{}).Name())
62 }
63}
64
65func TestPythonExecArgvDirect(t *testing.T) {
66 argv := pythonExecArgv("/usr/bin/python3")
67 if len(argv) != 4 || argv[0] != "/usr/bin/python3" || argv[1] != "-u" || argv[2] != "-B" || argv[3] != "-" {
68 t.Fatalf("argv = %q, want [exe -u -B -]", argv)
69 }
70}
71
72func TestPythonRejectsInvalidArgs(t *testing.T) {
73 tool := stubPythonTool(resolvedPython{}, nil, pythonExecOutcome{}, nil)
74 cases := []struct {
75 name string
76 raw json.RawMessage
77 want string
78 }{
79 {"blank code", mustJSON(t, map[string]any{"code": " ", "timeout": 5}), "code is required"},
80 {"missing timeout", mustJSON(t, map[string]any{"code": "print(1)"}), "timeout is required"},
81 {"zero timeout", mustJSON(t, map[string]any{"code": "print(1)", "timeout": 0}), "timeout must be between 1 and"},
82 {"negative timeout", mustJSON(t, map[string]any{"code": "print(1)", "timeout": -2}), "timeout must be between 1 and"},
83 }
84 for _, tc := range cases {
85 t.Run(tc.name, func(t *testing.T) {
86 res, err := tool.Execute(context.Background(), tc.raw, nil)
87 if err != nil {
88 t.Fatalf("err = %v, want model-visible IsError result", err)
89 }
90 if !res.IsError {
91 t.Fatal("want IsError for invalid args")
92 }
93 if got := res.Content[0].(provider.TextBlock).Text; !strings.Contains(got, tc.want) {
94 t.Fatalf("got %q want %q", got, tc.want)
95 }
96 })
97 }
98}
99
100func TestPythonDeniedNeverResolves(t *testing.T) {
101 resolveCalls := 0
102 execCalls := 0
103 mk := func(sandbox *Sandbox) *PythonTool {
104 return &PythonTool{
105 CWD: "/session",
106 Sandbox: sandbox,
107 resolveHook: func(context.Context, string, string) (resolvedPython, error) {
108 resolveCalls++
109 return resolvedPython{Path: "/usr/bin/python3"}, nil
110 },
111 execHook: func(context.Context, string, string, string, func(string)) (pythonExecOutcome, error) {
112 execCalls++
113 return pythonExecOutcome{}, nil
114 },
115 }
116 }
117 jailed := NewSandbox("/session")
118 jailed.Lock()
119 res, err := mk(jailed).Execute(context.Background(), pythonArgsJSON(t, "print(1)", 5), nil)
120 if err != nil {
121 t.Fatalf("jail err = %v, want model-visible IsError result", err)
122 }
123 if !res.IsError {
124 t.Fatal("want IsError for jailed execution")
125 }
126 if got := res.Content[0].(provider.TextBlock).Text; !strings.Contains(got, "cannot be confined") {
127 t.Fatalf("jail result = %q", got)
128 }
129 restricted := NewSandbox("/session")
130 restricted.SetPermissions(&PermissionSet{})
131 res, err = mk(restricted).Execute(context.Background(), pythonArgsJSON(t, "print(1)", 5), nil)
132 if err != nil {
133 t.Fatalf("permission err = %v, want model-visible IsError result", err)
134 }
135 if !res.IsError {
136 t.Fatal("want IsError for permission-denied execution")
137 }
138 if got := res.Content[0].(provider.TextBlock).Text; !strings.Contains(got, "permission denied") {
139 t.Fatalf("permission result = %q", got)
140 }
141 if resolveCalls != 0 || execCalls != 0 {
142 t.Fatalf("denied calls must never resolve or execute (resolve=%d exec=%d)", resolveCalls, execCalls)
143 }
144}
145
146func TestPythonMissingInterpreterActionable(t *testing.T) {
147 tool := &PythonTool{
148 CWD: "/session",
149 resolveHook: func(context.Context, string, string) (resolvedPython, error) {
150 return resolvedPython{}, missingPythonError()
151 },
152 execHook: func(context.Context, string, string, string, func(string)) (pythonExecOutcome, error) {
153 t.Fatal("missing interpreter must not execute")
154 return pythonExecOutcome{}, nil
155 },
156 }
157 res, err := tool.Execute(context.Background(), pythonArgsJSON(t, "print(1)", 5), nil)
158 if err != nil {
159 t.Fatalf("err = %v, want model-visible IsError result", err)
160 }
161 if !res.IsError {
162 t.Fatal("want IsError for missing interpreter")
163 }
164 got := res.Content[0].(provider.TextBlock).Text
165 if !strings.Contains(got, "existing Python 3") || !strings.Contains(got, "never installs") {
166 t.Fatalf("result = %q, want actionable guidance", got)
167 }
168}
169
170func TestPythonSuccessResult(t *testing.T) {
171 interp := resolvedPython{Path: "/usr/bin/python3", Version: pythonVersion{Major: 3, Minor: 12, Micro: 1}}
172 tool := stubPythonTool(interp, nil, pythonExecOutcome{Captured: "hi\n", ExitCode: 0}, nil)
173 res, err := tool.Execute(context.Background(), pythonArgsJSON(t, "print('hi')", 5), nil)
174 if err != nil {
175 t.Fatal(err)
176 }
177 if res.IsError {
178 t.Fatal("unexpected IsError")
179 }
180 text := res.Content[0].(provider.TextBlock).Text
181 for _, want := range []string{"python 3.12.1", "/usr/bin/python3", "hi", "[exit 0]"} {
182 if !strings.Contains(text, want) {
183 t.Fatalf("result missing %q:\n%s", want, text)
184 }
185 }
186 details := res.Details.(map[string]any)
187 if details["exit_code"] != 0 || details["interpreter"] != "/usr/bin/python3" || details["python_version"] != "3.12.1" || details["bytes_truncated"] != false || details["lines_truncated"] != false {
188 t.Fatalf("details = %#v", details)
189 }
190 if path, _ := details["full_output_path"].(string); path != "" || strings.Contains(text, "full output") {
191 t.Fatalf("untruncated result must not persist a full-output file: %q", path)
192 }
193}
194
195func TestPythonFailureStatuses(t *testing.T) {
196 interp := resolvedPython{Path: "/py", Version: pythonVersion{Major: 3}}
197 for _, tc := range []struct {
198 name string
199 outcome pythonExecOutcome
200 want string
201 }{
202 {"nonzero", pythonExecOutcome{Captured: "boom\n", ExitCode: 3}, "[exit 3]"},
203 {"timeout", pythonExecOutcome{Captured: "", ExitCode: -1, TimedOut: true}, "[timed out after 5 seconds]"},
204 {"cancelled", pythonExecOutcome{Captured: "", ExitCode: -1, Cancelled: true}, "[cancelled]"},
205 } {
206 t.Run(tc.name, func(t *testing.T) {
207 tool := stubPythonTool(interp, nil, tc.outcome, nil)
208 res, err := tool.Execute(context.Background(), pythonArgsJSON(t, "x", 5), nil)
209 if err != nil {
210 t.Fatal(err)
211 }
212 if !res.IsError {
213 t.Fatal("want IsError")
214 }
215 if got := res.Content[0].(provider.TextBlock).Text; !strings.Contains(got, tc.want) {
216 t.Fatalf("got %q want %q", got, tc.want)
217 }
218 })
219 }
220}
221
222func TestPythonCancelErrorCapture(t *testing.T) {
223 var ce pythonCancelError
224 if err := ce.get(); err != nil {
225 t.Fatalf("empty capture = %v, want nil", err)
226 }
227 ce.set(errors.New("kill group: boom"))
228 if err := ce.get(); err == nil || !strings.Contains(err.Error(), "boom") {
229 t.Fatalf("captured = %v, want kill error", err)
230 }
231 // A nil receiver must not panic: probes pass nil explicitly.
232 var nilCE *pythonCancelError
233 nilCE.set(errors.New("ignored"))
234 if err := nilCE.get(); err != nil {
235 t.Fatalf("nil capture = %v, want nil", err)
236 }
237}
238
239func TestPythonTruncationMarkedWithFullOutput(t *testing.T) {
240 interp := resolvedPython{Path: "/py", Version: pythonVersion{Major: 3, Minor: 1, Micro: 0}}
241 // Exactly maxPythonLines newline-terminated lines must not count as
242 // truncated: the trailing newline terminates the last line.
243 exact := strings.Repeat("x\n", maxPythonLines)
244 tool := stubPythonTool(interp, nil, pythonExecOutcome{Captured: exact, ExitCode: 0}, nil)
245 res, err := tool.Execute(context.Background(), pythonArgsJSON(t, "x", 5), nil)
246 if err != nil {
247 t.Fatal(err)
248 }
249 if got := res.Content[0].(provider.TextBlock).Text; strings.Contains(got, "truncated at") {
250 t.Fatalf("exact line budget must not truncate:\n%s", got)
251 }
252 if res.Details.(map[string]any)["lines_truncated"] != false {
253 t.Fatalf("details = %#v", res.Details)
254 }
255 if path, _ := res.Details.(map[string]any)["full_output_path"].(string); path != "" {
256 t.Fatalf("untruncated result must not persist an artifact: %q", path)
257 }
258
259 long := strings.Repeat("x\n", maxPythonLines+10)
260 tool = stubPythonTool(interp, nil, pythonExecOutcome{Captured: long, ExitCode: 0}, nil)
261 res, err = tool.Execute(context.Background(), pythonArgsJSON(t, "x", 5), nil)
262 if err != nil {
263 t.Fatal(err)
264 }
265 text := res.Content[0].(provider.TextBlock).Text
266 if !strings.Contains(text, "truncated at 2000 lines") {
267 t.Fatalf("lines truncation not marked:\n%s", text[len(text)-500:])
268 }
269 details := res.Details.(map[string]any)
270 if details["lines_truncated"] != true {
271 t.Fatalf("details = %#v", res.Details)
272 }
273 fullPath, _ := details["full_output_path"].(string)
274 if fullPath == "" || !strings.Contains(text, "full output: "+fullPath) {
275 t.Fatalf("truncated result must reference a full-output file: %q\n%s", fullPath, text)
276 }
277 t.Cleanup(func() { _ = os.Remove(fullPath) })
278 persisted, err := os.ReadFile(fullPath)
279 if err != nil {
280 t.Fatal(err)
281 }
282 if string(persisted) != long {
283 t.Fatal("full-output file must hold the byte-capped buffer beyond line trimming")
284 }
285
286 big := strings.Repeat("y", maxPythonBytes+100)
287 // The stub bypasses capture, so trim the fixture the way the real
288 // runner would: byte-capped, then line-trimmed for display.
289 big = big[:maxPythonBytes]
290 tool = stubPythonTool(interp, nil, pythonExecOutcome{Captured: big, ExitCode: 0, BytesTrunc: true}, nil)
291 res, err = tool.Execute(context.Background(), pythonArgsJSON(t, "x", 5), nil)
292 if err != nil {
293 t.Fatal(err)
294 }
295 if got := res.Content[0].(provider.TextBlock).Text; !strings.Contains(got, "truncated at 51200 bytes") {
296 t.Fatalf("bytes truncation not marked")
297 }
298 if path, _ := res.Details.(map[string]any)["full_output_path"].(string); path == "" {
299 t.Fatal("byte-truncated result must persist a full-output file")
300 } else {
301 t.Cleanup(func() { _ = os.Remove(path) })
302 }
303}
304
305func TestPythonProgressStreamsMergedOutput(t *testing.T) {
306 interp := resolvedPython{Path: "/py", Version: pythonVersion{Major: 3}}
307 var calls []string
308 tool := &PythonTool{
309 CWD: "/session",
310 resolveHook: func(context.Context, string, string) (resolvedPython, error) {
311 return interp, nil
312 },
313 execHook: func(_ context.Context, _, _, _ string, progress func(string)) (pythonExecOutcome, error) {
314 progress("out\n")
315 progress("err\n")
316 return pythonExecOutcome{Captured: "out\nerr\n", ExitCode: 0}, nil
317 },
318 }
319 res, err := tool.Execute(context.Background(), pythonArgsJSON(t, "x", 5), func(s string) {
320 calls = append(calls, s)
321 })
322 if err != nil {
323 t.Fatal(err)
324 }
325 if strings.Join(calls, "") != "out\nerr\n" {
326 t.Fatalf("progress = %q", calls)
327 }
328 if got := res.Content[0].(provider.TextBlock).Text; !strings.Contains(got, "out") || !strings.Contains(got, "err") {
329 t.Fatalf("merged output missing:\n%s", got)
330 }
331}
332
333func TestAppendPythonChunkBoundaries(t *testing.T) {
334 var buf bytes.Buffer
335 if appendPythonChunk(&buf, bytes.Repeat([]byte("y"), maxPythonBytes)) {
336 t.Fatal("output exactly filling the budget must not report truncation")
337 }
338 if buf.Len() != maxPythonBytes {
339 t.Fatalf("buffer len = %d, want %d", buf.Len(), maxPythonBytes)
340 }
341 // One byte over discards exactly that byte.
342 if !appendPythonChunk(&buf, []byte("!")) {
343 t.Fatal("overflow byte must report truncation")
344 }
345 if buf.Len() != maxPythonBytes || buf.Bytes()[buf.Len()-1] == '!' {
346 t.Fatal("overflow byte must not be retained")
347 }
348 // A chunk straddling the boundary keeps its fitting prefix and
349 // reports the discarded remainder.
350 buf.Reset()
351 if appendPythonChunk(&buf, bytes.Repeat([]byte("a"), maxPythonBytes-1)) {
352 t.Fatal("sub-budget write must not report truncation")
353 }
354 if !appendPythonChunk(&buf, []byte("bc")) {
355 t.Fatal("straddling write must report its discarded remainder")
356 }
357 if buf.Len() != maxPythonBytes || buf.Bytes()[buf.Len()-1] != 'b' {
358 t.Fatal("straddling write must retain only its fitting prefix")
359 }
360 if appendPythonChunk(&buf, nil) {
361 t.Fatal("empty chunk must not report truncation")
362 }
363}
364
365func TestPythonTimeoutBounds(t *testing.T) {
366 if _, err := pythonTimeoutDuration(0); err == nil {
367 t.Fatal("want error for 0")
368 }
369 if _, err := pythonTimeoutDuration(maxPythonTimeoutSeconds + 1); err == nil {
370 t.Fatal("want error above maximum")
371 }
372 if d, err := pythonTimeoutDuration(2); err != nil || d != 2*time.Second {
373 t.Fatalf("d = %s, err = %v", d, err)
374 }
375}
376
377func TestPythonResolutionSharesTimeoutBudget(t *testing.T) {
378 interp := resolvedPython{Path: "/py", Version: pythonVersion{Major: 3}}
379 tool := &PythonTool{
380 CWD: "/session",
381 resolveHook: func(ctx context.Context, _ string, _ string) (resolvedPython, error) {
382 if _, ok := ctx.Deadline(); !ok {
383 t.Fatal("resolution must share the invocation deadline")
384 }
385 return interp, nil
386 },
387 execHook: func(ctx context.Context, _, _, _ string, _ func(string)) (pythonExecOutcome, error) {
388 if _, ok := ctx.Deadline(); !ok {
389 t.Fatal("execution must share the invocation deadline")
390 }
391 return pythonExecOutcome{ExitCode: 0}, nil
392 },
393 }
394 if _, err := tool.Execute(context.Background(), pythonArgsJSON(t, "x", 5), nil); err != nil {
395 t.Fatal(err)
396 }
397}
398
399func TestPythonExecErrorPropagates(t *testing.T) {
400 tool := stubPythonTool(
401 resolvedPython{Path: "/py", Version: pythonVersion{Major: 3}},
402 nil, pythonExecOutcome{}, errors.New("start: boom"),
403 )
404 if _, err := tool.Execute(context.Background(), pythonArgsJSON(t, "x", 5), nil); err == nil || !strings.Contains(err.Error(), "boom") {
405 t.Fatalf("err = %v", err)
406 }
407}