fix(provider): clamp max_tokens to fit OpenRouter provider context window
- it currently rejects requests where input + max_output exceeds the serving provider's context lmit (which may be tighter than the model-level value) - use the smaller of ContextWindow and MaxOutput as the cap, with a 4096-token input reserve
- commit
- 1425e68636b67c73aaa224c138e098318be4d6d6
- author
- Neil Vallecer · · 2026-06-09 15:25 UTC
- parent
- 3d031dd
- signature
- unsigned
2 files · +27 -0
M packages/provider/discover.go +2 -0
| @@ -286,6 +286,8 @@ func DiscoverOpenRouter(ctx context.Context, baseURL string) ([]Model, error) { | |||
| 286 | 286 | ctxWin := d.ContextLength | |
| 287 | 287 | if ctxWin == 0 { | |
| 288 | 288 | ctxWin = d.TopProvider.ContextLength | |
| 289 | + | } else if d.TopProvider.ContextLength > 0 && d.TopProvider.ContextLength < ctxWin { | |
| 290 | + | ctxWin = d.TopProvider.ContextLength | |
| 289 | 291 | } | |
| 290 | 292 | maxOut := 0 | |
| 291 | 293 | if d.TopProvider.MaxCompletionTokens != nil { | |
| @@ -286,6 +286,8 @@ func DiscoverOpenRouter(ctx context.Context, baseURL string) ([]Model, error) { | |||||
| 286 | ctxWin := d.ContextLength | 286 | ctxWin := d.ContextLength | ||
| 287 | if ctxWin == 0 { | 287 | if ctxWin == 0 { | ||
| 288 | ctxWin = d.TopProvider.ContextLength | 288 | ctxWin = d.TopProvider.ContextLength | ||
| 289 | + | } else if d.TopProvider.ContextLength > 0 && d.TopProvider.ContextLength < ctxWin { | |||
| 290 | + | ctxWin = d.TopProvider.ContextLength | |||
| 289 | } | 291 | } | ||
| 290 | maxOut := 0 | 292 | maxOut := 0 | ||
| 291 | if d.TopProvider.MaxCompletionTokens != nil { | 293 | if d.TopProvider.MaxCompletionTokens != nil { | ||
M packages/provider/openai.go +25 -0
| @@ -183,6 +183,31 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||
| 183 | 183 | if maxTok <= 0 { | |
| 184 | 184 | maxTok = m.MaxOutput | |
| 185 | 185 | } | |
| 186 | + | // Clamp max_tokens so output + minimum input fits within the context | |
| 187 | + | // window. Some providers (OpenRouter) enforce input + max_output <= | |
| 188 | + | // context_length and reject requests where the total exceeds it. | |
| 189 | + | // This reservation ensures basic user input (system prompt + first | |
| 190 | + | // message + tool definitions) has room. | |
| 191 | + | // | |
| 192 | + | // Use the smaller of ContextWindow and MaxOutput as the cap because | |
| 193 | + | // some providers (OpenRouter) report inflated model-level context | |
| 194 | + | // windows (e.g. 1000000) while the serving provider enforces a much | |
| 195 | + | // tighter limit (e.g. 262144). When both are wrong we still land on | |
| 196 | + | // the output cap, which is the actual limit reported by the provider. | |
| 197 | + | if m.ContextWindow > 0 && m.MaxOutput > 0 { | |
| 198 | + | ctxCap := m.ContextWindow | |
| 199 | + | if m.MaxOutput < ctxCap { | |
| 200 | + | ctxCap = m.MaxOutput | |
| 201 | + | } | |
| 202 | + | const reserve = 4096 | |
| 203 | + | clamped := ctxCap - reserve | |
| 204 | + | if clamped < 1024 { | |
| 205 | + | clamped = 1024 | |
| 206 | + | } | |
| 207 | + | if maxTok > clamped { | |
| 208 | + | maxTok = clamped | |
| 209 | + | } | |
| 210 | + | } | |
| 186 | 211 | if m.Reasoning { | |
| 187 | 212 | if maxTok > 0 { | |
| 188 | 213 | out.MaxCompletionTok = &maxTok | |
| @@ -183,6 +183,31 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||||
| 183 | if maxTok <= 0 { | 183 | if maxTok <= 0 { | ||
| 184 | maxTok = m.MaxOutput | 184 | maxTok = m.MaxOutput | ||
| 185 | } | 185 | } | ||
| 186 | + | // Clamp max_tokens so output + minimum input fits within the context | |||
| 187 | + | // window. Some providers (OpenRouter) enforce input + max_output <= | |||
| 188 | + | // context_length and reject requests where the total exceeds it. | |||
| 189 | + | // This reservation ensures basic user input (system prompt + first | |||
| 190 | + | // message + tool definitions) has room. | |||
| 191 | + | // | |||
| 192 | + | // Use the smaller of ContextWindow and MaxOutput as the cap because | |||
| 193 | + | // some providers (OpenRouter) report inflated model-level context | |||
| 194 | + | // windows (e.g. 1000000) while the serving provider enforces a much | |||
| 195 | + | // tighter limit (e.g. 262144). When both are wrong we still land on | |||
| 196 | + | // the output cap, which is the actual limit reported by the provider. | |||
| 197 | + | if m.ContextWindow > 0 && m.MaxOutput > 0 { | |||
| 198 | + | ctxCap := m.ContextWindow | |||
| 199 | + | if m.MaxOutput < ctxCap { | |||
| 200 | + | ctxCap = m.MaxOutput | |||
| 201 | + | } | |||
| 202 | + | const reserve = 4096 | |||
| 203 | + | clamped := ctxCap - reserve | |||
| 204 | + | if clamped < 1024 { | |||
| 205 | + | clamped = 1024 | |||
| 206 | + | } | |||
| 207 | + | if maxTok > clamped { | |||
| 208 | + | maxTok = clamped | |||
| 209 | + | } | |||
| 210 | + | } | |||
| 186 | if m.Reasoning { | 211 | if m.Reasoning { | ||
| 187 | if maxTok > 0 { | 212 | if maxTok > 0 { | ||
| 188 | out.MaxCompletionTok = &maxTok | 213 | out.MaxCompletionTok = &maxTok | ||