openrouter_server_tools_live_test.go RawHistoryFolder
1package provider
2
3import (
4 "bytes"
5 "context"
6 "encoding/json"
7 "fmt"
8 "io"
9 "net/http"
10 "os"
11 "strings"
12 "testing"
13 "time"
14)
15
16const openRouterLiveGeneralModel = "openai/gpt-4.1"
17
18// Live OpenRouter server-tool batteries. Skipped unless OPENROUTER_API_KEY
19// is set so CI and unpaid checkouts never hit the network.
20func TestOpenRouterServerToolsLiveDeepSeekFlash(t *testing.T) {
21 runOpenRouterServerToolBattery(t, "deepseek/deepseek-v4-flash")
22}
23
24func TestOpenRouterServerToolsLivePresetFlash(t *testing.T) {
25 runOpenRouterServerToolBattery(t, "@preset/flash")
26}
27
28func TestOpenRouterServerToolsLiveGeneral(t *testing.T) {
29 runOpenRouterServerToolBatteryOnNativeAPIs(t, openRouterLiveGeneralModel)
30}
31
32func TestOpenRouterServerToolsLiveGeneralWithSettingsEnabled(t *testing.T) {
33 // Same multidisciplinary model as the settings-on agent path. Chat
34 // Completions-capable tools are advertised together (the setting's
35 // default-on registry); shell/bash/apply_patch/tool_search still use
36 // their native APIs.
37 if testing.Short() {
38 t.Skip("skip live OpenRouter battery in -short mode")
39 }
40 key := strings.TrimSpace(os.Getenv("OPENROUTER_API_KEY"))
41 if key == "" {
42 t.Skip("OPENROUTER_API_KEY not set")
43 }
44
45 client := NewOpenRouter(key, "")
46 settingsTools := make([]Tool, 0, len(openRouterServerToolCases))
47 for _, tc := range openRouterServerToolCases {
48 if !tc.chatCompletionsUnsupported {
49 settingsTools = append(settingsTools, tc.tool)
50 }
51 }
52
53 for _, tc := range openRouterServerToolCases {
54 tc := tc
55 t.Run(tc.name, func(t *testing.T) {
56 ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
57 defer cancel()
58 switch tc.name {
59 case "openrouter:shell", "openrouter:apply_patch", "openrouter:tool_search":
60 postOpenRouterJSON(t, ctx, key, openrouterDefaultBaseURL+"/responses", nativeResponsesBody(openRouterLiveGeneralModel, tc))
61 case "openrouter:bash":
62 postOpenRouterJSON(t, ctx, key, openrouterDefaultBaseURL+"/messages", nativeMessagesBody(openRouterLiveGeneralModel, tc))
63 default:
64 stream, err := client.Stream(ctx, Request{
65 Model: openRouterLiveGeneralModel,
66 Tools: settingsTools,
67 MaxTokens: 256,
68 MaxToolCalls: 4,
69 SessionID: "zot-live-settings-" + strings.TrimPrefix(tc.name, "openrouter:"),
70 Messages: []Message{{
71 Role: RoleUser,
72 Content: []Content{TextBlock{Text: tc.prompt}},
73 }},
74 })
75 if err != nil {
76 t.Fatalf("Stream(%s): %v", tc.name, err)
77 }
78 var done EventDone
79 for ev := range stream {
80 if e, ok := ev.(EventDone); ok {
81 done = e
82 }
83 }
84 if done.Err != nil {
85 t.Fatalf("EventDone.Err for %s: %v", tc.name, done.Err)
86 }
87 if done.Stop == StopError {
88 t.Fatalf("stop=error for %s", tc.name)
89 }
90 }
91 })
92 }
93}
94
95type openRouterServerToolCase struct {
96 name string
97 tool Tool
98 prompt string
99 // chatCompletionsUnsupported is true when OpenRouter documents the
100 // tool as Chat Completions-incompatible (400).
101 chatCompletionsUnsupported bool
102}
103
104var openRouterServerToolCases = []openRouterServerToolCase{
105 {
106 name: "openrouter:web_search",
107 tool: Tool{Name: "openrouter:web_search", Schema: json.RawMessage(`{"max_results":1}`)},
108 prompt: "Search the web for 'OpenRouter AI'. You MUST call the web search tool, then reply with one sentence.",
109 },
110 {
111 name: "openrouter:web_fetch",
112 tool: Tool{Name: "openrouter:web_fetch"},
113 prompt: "Fetch https://example.com and quote the h1. You MUST call the web fetch tool.",
114 },
115 {
116 name: "openrouter:advisor",
117 tool: Tool{Name: "openrouter:advisor", Schema: json.RawMessage(`{"model":"openai/gpt-4.1-mini"}`)},
118 prompt: "Ask the advisor whether 17 is prime. You MUST call the advisor tool, then answer in one word.",
119 },
120 {
121 name: "openrouter:subagent",
122 tool: Tool{Name: "openrouter:subagent", Schema: json.RawMessage(`{"model":"openai/gpt-4.1-mini"}`)},
123 prompt: "Delegate to a subagent: return the sum of 21 and 21. You MUST call the subagent tool.",
124 },
125 {
126 name: "openrouter:fusion",
127 tool: Tool{Name: "openrouter:fusion", Schema: json.RawMessage(`{"analysis_models":["openai/gpt-4.1-mini"],"model":"openai/gpt-4.1-mini"}`)},
128 prompt: "Use fusion to answer in one sentence: what is 1+1? You MUST call the fusion tool.",
129 },
130 {
131 name: "openrouter:shell",
132 tool: Tool{Name: "openrouter:shell"},
133 prompt: "Run `echo hi` in the hosted shell. You MUST call the shell tool.",
134 chatCompletionsUnsupported: true,
135 },
136 {
137 name: "openrouter:bash",
138 tool: Tool{Name: "openrouter:bash"},
139 prompt: "Run `echo hi` with bash. You MUST call the bash tool.",
140 chatCompletionsUnsupported: true,
141 },
142 {
143 name: "openrouter:apply_patch",
144 tool: Tool{Name: "openrouter:apply_patch"},
145 prompt: "Propose a patch that creates hello.txt with hello. You MUST call apply_patch.",
146 chatCompletionsUnsupported: true,
147 },
148 {
149 name: "openrouter:image_generation",
150 tool: Tool{Name: "openrouter:image_generation"},
151 prompt: "Generate a tiny image of a red square. You MUST call image generation, then describe the result in one sentence.",
152 },
153 {
154 name: "openrouter:datetime",
155 tool: Tool{Name: "openrouter:datetime"},
156 prompt: "What is the current UTC datetime? You MUST call the datetime tool and then print only the datetime.",
157 },
158 {
159 name: "openrouter:tool_search",
160 tool: Tool{Name: "openrouter:tool_search"},
161 prompt: "Find a tool that can add numbers. You MUST call tool_search first.",
162 chatCompletionsUnsupported: true,
163 },
164 {
165 name: "openrouter:experimental__search_models",
166 tool: Tool{Name: "openrouter:experimental__search_models", Schema: json.RawMessage(`{"max_results":3}`)},
167 prompt: "Search the OpenRouter catalog for cheap flash models. You MUST call the search models tool and list up to 3 ids.",
168 },
169}
170
171func runOpenRouterServerToolBattery(t *testing.T, model string) {
172 t.Helper()
173 if testing.Short() {
174 t.Skip("skip live OpenRouter battery in -short mode")
175 }
176 key := strings.TrimSpace(os.Getenv("OPENROUTER_API_KEY"))
177 if key == "" {
178 t.Skip("OPENROUTER_API_KEY not set")
179 }
180
181 client := NewOpenRouter(key, "")
182 for _, tc := range openRouterServerToolCases {
183 tc := tc
184 t.Run(tc.name, func(t *testing.T) {
185 ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
186 defer cancel()
187
188 tools := []Tool{tc.tool}
189 if tc.name == "openrouter:tool_search" {
190 tools = append(tools, Tool{
191 Name: "add_numbers",
192 Description: "Add two integers.",
193 Schema: json.RawMessage(`{"type":"object","properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"]}`),
194 Deferred: true,
195 })
196 }
197
198 stream, err := client.Stream(ctx, Request{
199 Model: model,
200 Tools: tools,
201 MaxTokens: 256,
202 MaxToolCalls: 4,
203 SessionID: "zot-live-" + strings.ReplaceAll(model, "/", "-") + "-" + strings.TrimPrefix(tc.name, "openrouter:"),
204 Messages: []Message{{
205 Role: RoleUser,
206 Content: []Content{TextBlock{Text: tc.prompt}},
207 }},
208 })
209 if tc.chatCompletionsUnsupported {
210 if err == nil {
211 // Drain so the HTTP body is not leaked if the
212 // server accepted the request anyway.
213 for range stream {
214 }
215 t.Fatalf("expected chat-completions rejection for %s, got stream", tc.name)
216 }
217 if !strings.Contains(strings.ToLower(err.Error()), "chat-completions") &&
218 !strings.Contains(strings.ToLower(err.Error()), "not available") {
219 t.Fatalf("unexpected error for unsupported tool %s: %v", tc.name, err)
220 }
221 return
222 }
223 if err != nil {
224 t.Fatalf("Stream(%s, %s): %v", model, tc.name, err)
225 }
226 var done EventDone
227 for ev := range stream {
228 if e, ok := ev.(EventDone); ok {
229 done = e
230 }
231 }
232 if done.Err != nil {
233 t.Fatalf("EventDone.Err for %s/%s: %v", model, tc.name, done.Err)
234 }
235 if done.Stop == StopError {
236 t.Fatalf("stop=error for %s/%s", model, tc.name)
237 }
238 })
239 }
240}
241
242func runOpenRouterServerToolBatteryOnNativeAPIs(t *testing.T, model string) {
243 t.Helper()
244 if testing.Short() {
245 t.Skip("skip live OpenRouter battery in -short mode")
246 }
247 key := strings.TrimSpace(os.Getenv("OPENROUTER_API_KEY"))
248 if key == "" {
249 t.Skip("OPENROUTER_API_KEY not set")
250 }
251
252 client := NewOpenRouter(key, "")
253 for _, tc := range openRouterServerToolCases {
254 tc := tc
255 t.Run(tc.name, func(t *testing.T) {
256 ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
257 defer cancel()
258 switch tc.name {
259 case "openrouter:shell", "openrouter:apply_patch", "openrouter:tool_search":
260 postOpenRouterJSON(t, ctx, key, openrouterDefaultBaseURL+"/responses", nativeResponsesBody(model, tc))
261 case "openrouter:bash":
262 postOpenRouterJSON(t, ctx, key, openrouterDefaultBaseURL+"/messages", nativeMessagesBody(model, tc))
263 default:
264 runOpenRouterChatTool(t, ctx, client, model, tc)
265 }
266 })
267 }
268}
269
270func runOpenRouterChatTool(t *testing.T, ctx context.Context, client Client, model string, tc openRouterServerToolCase) {
271 t.Helper()
272 stream, err := client.Stream(ctx, Request{
273 Model: model,
274 Tools: []Tool{tc.tool},
275 MaxTokens: 256,
276 MaxToolCalls: 4,
277 SessionID: "zot-live-general-" + strings.TrimPrefix(tc.name, "openrouter:"),
278 Messages: []Message{{
279 Role: RoleUser,
280 Content: []Content{TextBlock{Text: tc.prompt}},
281 }},
282 })
283 if err != nil {
284 t.Fatalf("Stream(%s, %s): %v", model, tc.name, err)
285 }
286 var done EventDone
287 for ev := range stream {
288 if e, ok := ev.(EventDone); ok {
289 done = e
290 }
291 }
292 if done.Err != nil {
293 t.Fatalf("EventDone.Err for %s/%s: %v", model, tc.name, done.Err)
294 }
295 if done.Stop == StopError {
296 t.Fatalf("stop=error for %s/%s", model, tc.name)
297 }
298}
299
300func nativeResponsesBody(model string, tc openRouterServerToolCase) map[string]any {
301 tool := map[string]any{"type": tc.name}
302 switch tc.name {
303 case "openrouter:shell":
304 tool["parameters"] = map[string]any{"engine": "openrouter"}
305 case "openrouter:tool_search":
306 return map[string]any{
307 "model": model,
308 "input": tc.prompt,
309 "tools": []any{
310 map[string]any{"type": "openrouter:tool_search"},
311 map[string]any{
312 "type": "function",
313 "name": "add_numbers",
314 "description": "Add two integers.",
315 "parameters": map[string]any{
316 "type": "object",
317 "properties": map[string]any{"a": map[string]any{"type": "integer"}, "b": map[string]any{"type": "integer"}},
318 "required": []string{"a", "b"},
319 },
320 "defer_loading": true,
321 },
322 },
323 }
324 }
325 return map[string]any{
326 "model": model,
327 "input": tc.prompt,
328 "tools": []any{tool},
329 }
330}
331
332func nativeMessagesBody(model string, tc openRouterServerToolCase) map[string]any {
333 return map[string]any{
334 "model": model,
335 "max_tokens": 256,
336 "messages": []any{map[string]any{"role": "user", "content": tc.prompt}},
337 "tools": []any{map[string]any{
338 "type": tc.name,
339 "parameters": map[string]any{"engine": "openrouter"},
340 }},
341 }
342}
343
344func postOpenRouterJSON(t *testing.T, ctx context.Context, key, url string, body map[string]any) {
345 t.Helper()
346 payload, err := json.Marshal(body)
347 if err != nil {
348 t.Fatal(err)
349 }
350 req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
351 if err != nil {
352 t.Fatal(err)
353 }
354 req.Header.Set("Authorization", "Bearer "+key)
355 req.Header.Set("Content-Type", "application/json")
356 resp, err := http.DefaultClient.Do(req)
357 if err != nil {
358 t.Fatalf("%s: %v", url, err)
359 }
360 defer resp.Body.Close()
361 raw, _ := io.ReadAll(resp.Body)
362 if resp.StatusCode != http.StatusOK {
363 t.Fatalf("%s http %d: %s", url, resp.StatusCode, truncateLiveBody(raw))
364 }
365 var decoded map[string]any
366 if err := json.Unmarshal(raw, &decoded); err != nil {
367 t.Fatalf("decode %s: %v\n%s", url, err, truncateLiveBody(raw))
368 }
369 if errObj, ok := decoded["error"]; ok && errObj != nil {
370 t.Fatalf("%s error: %v", url, errObj)
371 }
372}
373
374func truncateLiveBody(raw []byte) string {
375 s := strings.TrimSpace(string(raw))
376 if len(s) > 400 {
377 return s[:400] + fmt.Sprintf("... (%d bytes)", len(s))
378 }
379 return s
380}