fix(provider): account for DeepSeek cache hits (#209)
- commit
- 969e52a29a66e74ea5936b9d920920d9482e94dc
- author
- bnema · · 2026-09-14 19:45 UTC
- committer
- GitHub · · 2026-09-14 19:45 UTC
- parent
- e6deef9
- signature
- GPG Unverified
B5690EEEBB952194The key is unknown - refs
- tag: v0.38.1
3 files · +63 -3
M packages/provider/openai.go +13 -3
| @@ -322,7 +322,7 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||
| 322 | 322 | // message to a plain string and silently drop image blocks for | |
| 323 | 323 | // this provider so historical sessions with screenshots still replay. | |
| 324 | 324 | // The same quirk applies to DeepSeek models routed through opencode-go. | |
| 325 | - | textOnly := c.name == ProviderDeepSeek || (c.name == ProviderOpenCodeGo && isDeepSeekModel(req.Model)) | |
| 325 | + | textOnly := c.isDeepSeekRoute(req.Model) | |
| 326 | 326 | ||
| 327 | 327 | req.Messages = RepairOrphanedToolResults(req.Messages) | |
| 328 | 328 | for msgIndex := 0; msgIndex < len(req.Messages); msgIndex++ { | |
| @@ -332,7 +332,7 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||
| 332 | 332 | case RoleDeveloper: | |
| 333 | 333 | content := buildOAIUserContent(msg.Content, textOnly) | |
| 334 | 334 | role := "developer" | |
| 335 | - | if c.name == ProviderDeepSeek || (c.name == ProviderOpenCodeGo && isDeepSeekModel(req.Model)) { | |
| 335 | + | if c.isDeepSeekRoute(req.Model) { | |
| 336 | 336 | role = "system" | |
| 337 | 337 | } | |
| 338 | 338 | out.Messages = append(out.Messages, oaiMessage{Role: role, Content: content}) | |
| @@ -477,6 +477,10 @@ func supportsDeferredTools(provider string) bool { | |||
| 477 | 477 | return provider == "moonshotai" || provider == "moonshotai-cn" | |
| 478 | 478 | } | |
| 479 | 479 | ||
| 480 | + | func (c *openaiClient) isDeepSeekRoute(model string) bool { | |
| 481 | + | return c.name == ProviderDeepSeek || (c.name == ProviderOpenCodeGo && isDeepSeekModel(model)) | |
| 482 | + | } | |
| 483 | + | ||
| 480 | 484 | func isDeepSeekModel(model string) bool { | |
| 481 | 485 | model = strings.ToLower(strings.TrimSpace(model)) | |
| 482 | 486 | return strings.HasPrefix(model, "deepseek-") || strings.Contains(model, "/deepseek-") | |
| @@ -759,6 +763,8 @@ func (c *openaiClient) runStream(ctx context.Context, resp *http.Response, req R | |||
| 759 | 763 | } `json:"choices"` | |
| 760 | 764 | Usage *struct { | |
| 761 | 765 | PromptTokens int `json:"prompt_tokens"` | |
| 766 | + | PromptCacheHitTokens *int `json:"prompt_cache_hit_tokens"` | |
| 767 | + | PromptCacheMissTokens *int `json:"prompt_cache_miss_tokens"` | |
| 762 | 768 | CompletionTokens int `json:"completion_tokens"` | |
| 763 | 769 | PromptTokensDetails *openAIInputTokensDetails `json:"prompt_tokens_details"` | |
| 764 | 770 | CompletionTokensDetails *struct { | |
| @@ -780,7 +786,11 @@ func (c *openaiClient) runStream(ctx context.Context, resp *http.Response, req R | |||
| 780 | 786 | return | |
| 781 | 787 | } | |
| 782 | 788 | if chunk.Usage != nil { | |
| 783 | - | usage = normalizeOpenAIUsage(chunk.Usage.PromptTokens, chunk.Usage.CompletionTokens, chunk.Usage.PromptTokensDetails) | |
| 789 | + | if c.isDeepSeekRoute(req.Model) && (chunk.Usage.PromptCacheHitTokens != nil || chunk.Usage.PromptCacheMissTokens != nil) { | |
| 790 | + | usage = normalizeDeepSeekUsage(chunk.Usage.PromptTokens, chunk.Usage.CompletionTokens, chunk.Usage.PromptCacheHitTokens, chunk.Usage.PromptCacheMissTokens) | |
| 791 | + | } else { | |
| 792 | + | usage = normalizeOpenAIUsage(chunk.Usage.PromptTokens, chunk.Usage.CompletionTokens, chunk.Usage.PromptTokensDetails) | |
| 793 | + | } | |
| 784 | 794 | if details := chunk.Usage.CompletionTokensDetails; details != nil { | |
| 785 | 795 | usage.ReasoningTokens = details.ReasoningTokens | |
| 786 | 796 | usage.ReasoningTokensKnown = true | |
| @@ -322,7 +322,7 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||||
| 322 | // message to a plain string and silently drop image blocks for | 322 | // message to a plain string and silently drop image blocks for | ||
| 323 | // this provider so historical sessions with screenshots still replay. | 323 | // this provider so historical sessions with screenshots still replay. | ||
| 324 | // The same quirk applies to DeepSeek models routed through opencode-go. | 324 | // The same quirk applies to DeepSeek models routed through opencode-go. | ||
| 325 | - | textOnly := c.name == ProviderDeepSeek || (c.name == ProviderOpenCodeGo && isDeepSeekModel(req.Model)) | 325 | + | textOnly := c.isDeepSeekRoute(req.Model) |
| 326 | 326 | ||||
| 327 | req.Messages = RepairOrphanedToolResults(req.Messages) | 327 | req.Messages = RepairOrphanedToolResults(req.Messages) | ||
| 328 | for msgIndex := 0; msgIndex < len(req.Messages); msgIndex++ { | 328 | for msgIndex := 0; msgIndex < len(req.Messages); msgIndex++ { | ||
| @@ -332,7 +332,7 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||||
| 332 | case RoleDeveloper: | 332 | case RoleDeveloper: | ||
| 333 | content := buildOAIUserContent(msg.Content, textOnly) | 333 | content := buildOAIUserContent(msg.Content, textOnly) | ||
| 334 | role := "developer" | 334 | role := "developer" | ||
| 335 | - | if c.name == ProviderDeepSeek || (c.name == ProviderOpenCodeGo && isDeepSeekModel(req.Model)) { | 335 | + | if c.isDeepSeekRoute(req.Model) { |
| 336 | role = "system" | 336 | role = "system" | ||
| 337 | } | 337 | } | ||
| 338 | out.Messages = append(out.Messages, oaiMessage{Role: role, Content: content}) | 338 | out.Messages = append(out.Messages, oaiMessage{Role: role, Content: content}) | ||
| @@ -477,6 +477,10 @@ func supportsDeferredTools(provider string) bool { | |||||
| 477 | return provider == "moonshotai" || provider == "moonshotai-cn" | 477 | return provider == "moonshotai" || provider == "moonshotai-cn" | ||
| 478 | } | 478 | } | ||
| 479 | 479 | ||||
| 480 | + | func (c *openaiClient) isDeepSeekRoute(model string) bool { | |||
| 481 | + | return c.name == ProviderDeepSeek || (c.name == ProviderOpenCodeGo && isDeepSeekModel(model)) | |||
| 482 | + | } | |||
| 483 | + | ||||
| 480 | func isDeepSeekModel(model string) bool { | 484 | func isDeepSeekModel(model string) bool { | ||
| 481 | model = strings.ToLower(strings.TrimSpace(model)) | 485 | model = strings.ToLower(strings.TrimSpace(model)) | ||
| 482 | return strings.HasPrefix(model, "deepseek-") || strings.Contains(model, "/deepseek-") | 486 | return strings.HasPrefix(model, "deepseek-") || strings.Contains(model, "/deepseek-") | ||
| @@ -759,6 +763,8 @@ func (c *openaiClient) runStream(ctx context.Context, resp *http.Response, req R | |||||
| 759 | } `json:"choices"` | 763 | } `json:"choices"` | ||
| 760 | Usage *struct { | 764 | Usage *struct { | ||
| 761 | PromptTokens int `json:"prompt_tokens"` | 765 | PromptTokens int `json:"prompt_tokens"` | ||
| 766 | + | PromptCacheHitTokens *int `json:"prompt_cache_hit_tokens"` | |||
| 767 | + | PromptCacheMissTokens *int `json:"prompt_cache_miss_tokens"` | |||
| 762 | CompletionTokens int `json:"completion_tokens"` | 768 | CompletionTokens int `json:"completion_tokens"` | ||
| 763 | PromptTokensDetails *openAIInputTokensDetails `json:"prompt_tokens_details"` | 769 | PromptTokensDetails *openAIInputTokensDetails `json:"prompt_tokens_details"` | ||
| 764 | CompletionTokensDetails *struct { | 770 | CompletionTokensDetails *struct { | ||
| @@ -780,7 +786,11 @@ func (c *openaiClient) runStream(ctx context.Context, resp *http.Response, req R | |||||
| 780 | return | 786 | return | ||
| 781 | } | 787 | } | ||
| 782 | if chunk.Usage != nil { | 788 | if chunk.Usage != nil { | ||
| 783 | - | usage = normalizeOpenAIUsage(chunk.Usage.PromptTokens, chunk.Usage.CompletionTokens, chunk.Usage.PromptTokensDetails) | 789 | + | if c.isDeepSeekRoute(req.Model) && (chunk.Usage.PromptCacheHitTokens != nil || chunk.Usage.PromptCacheMissTokens != nil) { |
| 790 | + | usage = normalizeDeepSeekUsage(chunk.Usage.PromptTokens, chunk.Usage.CompletionTokens, chunk.Usage.PromptCacheHitTokens, chunk.Usage.PromptCacheMissTokens) | |||
| 791 | + | } else { | |||
| 792 | + | usage = normalizeOpenAIUsage(chunk.Usage.PromptTokens, chunk.Usage.CompletionTokens, chunk.Usage.PromptTokensDetails) | |||
| 793 | + | } | |||
| 784 | if details := chunk.Usage.CompletionTokensDetails; details != nil { | 794 | if details := chunk.Usage.CompletionTokensDetails; details != nil { | ||
| 785 | usage.ReasoningTokens = details.ReasoningTokens | 795 | usage.ReasoningTokens = details.ReasoningTokens | ||
| 786 | usage.ReasoningTokensKnown = true | 796 | usage.ReasoningTokensKnown = true | ||
M packages/provider/openai_usage.go +22 -0
| @@ -8,6 +8,28 @@ type openAIInputTokensDetails struct { | |||
| 8 | 8 | CacheWriteTokens int `json:"cache_write_tokens"` | |
| 9 | 9 | } | |
| 10 | 10 | ||
| 11 | + | // normalizeDeepSeekUsage converts DeepSeek's explicit hit/miss counts into | |
| 12 | + | // disjoint usage buckets. Cache accounting is accepted only when both fields | |
| 13 | + | // are present and add up to the reported prompt total. | |
| 14 | + | func normalizeDeepSeekUsage(inputTokens, outputTokens int, cacheHits, cacheMisses *int) Usage { | |
| 15 | + | usage := Usage{InputTokens: inputTokens, OutputTokens: outputTokens} | |
| 16 | + | if inputTokens < 0 || outputTokens < 0 || cacheHits == nil || cacheMisses == nil || *cacheHits < 0 || *cacheMisses < 0 || *cacheHits+*cacheMisses != inputTokens { | |
| 17 | + | if usage.InputTokens < 0 { | |
| 18 | + | usage.InputTokens = 0 | |
| 19 | + | } | |
| 20 | + | if usage.OutputTokens < 0 { | |
| 21 | + | usage.OutputTokens = 0 | |
| 22 | + | } | |
| 23 | + | return usage | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | usage.InputTokens = *cacheMisses | |
| 27 | + | usage.CacheReadTokens = *cacheHits | |
| 28 | + | usage.CacheMeasuredPromptTokens = inputTokens | |
| 29 | + | usage.CacheMeasuredReadTokens = *cacheHits | |
| 30 | + | return usage | |
| 31 | + | } | |
| 32 | + | ||
| 11 | 33 | // normalizeOpenAIUsage converts OpenAI's total input count and optional cache | |
| 12 | 34 | // detail into disjoint usage buckets. Malformed detail must not create a | |
| 13 | 35 | // negative ordinary-input count or an unsupported cache metric, so it falls | |
| @@ -8,6 +8,28 @@ type openAIInputTokensDetails struct { | |||||
| 8 | CacheWriteTokens int `json:"cache_write_tokens"` | 8 | CacheWriteTokens int `json:"cache_write_tokens"` | ||
| 9 | } | 9 | } | ||
| 10 | 10 | ||||
| 11 | + | // normalizeDeepSeekUsage converts DeepSeek's explicit hit/miss counts into | |||
| 12 | + | // disjoint usage buckets. Cache accounting is accepted only when both fields | |||
| 13 | + | // are present and add up to the reported prompt total. | |||
| 14 | + | func normalizeDeepSeekUsage(inputTokens, outputTokens int, cacheHits, cacheMisses *int) Usage { | |||
| 15 | + | usage := Usage{InputTokens: inputTokens, OutputTokens: outputTokens} | |||
| 16 | + | if inputTokens < 0 || outputTokens < 0 || cacheHits == nil || cacheMisses == nil || *cacheHits < 0 || *cacheMisses < 0 || *cacheHits+*cacheMisses != inputTokens { | |||
| 17 | + | if usage.InputTokens < 0 { | |||
| 18 | + | usage.InputTokens = 0 | |||
| 19 | + | } | |||
| 20 | + | if usage.OutputTokens < 0 { | |||
| 21 | + | usage.OutputTokens = 0 | |||
| 22 | + | } | |||
| 23 | + | return usage | |||
| 24 | + | } | |||
| 25 | + | ||||
| 26 | + | usage.InputTokens = *cacheMisses | |||
| 27 | + | usage.CacheReadTokens = *cacheHits | |||
| 28 | + | usage.CacheMeasuredPromptTokens = inputTokens | |||
| 29 | + | usage.CacheMeasuredReadTokens = *cacheHits | |||
| 30 | + | return usage | |||
| 31 | + | } | |||
| 32 | + | ||||
| 11 | // normalizeOpenAIUsage converts OpenAI's total input count and optional cache | 33 | // normalizeOpenAIUsage converts OpenAI's total input count and optional cache | ||
| 12 | // detail into disjoint usage buckets. Malformed detail must not create a | 34 | // detail into disjoint usage buckets. Malformed detail must not create a | ||
| 13 | // negative ordinary-input count or an unsupported cache metric, so it falls | 35 | // negative ordinary-input count or an unsupported cache metric, so it falls | ||
M packages/provider/reasoning_usage_test.go +28 -0
| @@ -28,6 +28,34 @@ func TestOpenAIReportsReasoningTokens(t *testing.T) { | |||
| 28 | 28 | } | |
| 29 | 29 | } | |
| 30 | 30 | ||
| 31 | + | func TestDeepSeekNormalizesNativeCacheUsage(t *testing.T) { | |
| 32 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 33 | + | w.Header().Set("content-type", "text/event-stream") | |
| 34 | + | fmt.Fprintln(w, `data: {"choices":[{"delta":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":100,"prompt_cache_hit_tokens":70,"prompt_cache_miss_tokens":30,"completion_tokens":9}}`) | |
| 35 | + | fmt.Fprintln(w) | |
| 36 | + | fmt.Fprintln(w, "data: [DONE]") | |
| 37 | + | fmt.Fprintln(w) | |
| 38 | + | })) | |
| 39 | + | defer srv.Close() | |
| 40 | + | ||
| 41 | + | events, err := NewDeepSeek("test", srv.URL).Stream(context.Background(), Request{Model: "deepseek-flash"}) | |
| 42 | + | if err != nil { | |
| 43 | + | t.Fatal(err) | |
| 44 | + | } | |
| 45 | + | usage := usageFromEvents(events) | |
| 46 | + | if usage.InputTokens != 30 || usage.CacheReadTokens != 70 || usage.CacheWriteTokens != 0 || usage.CacheMeasuredPromptTokens != 100 || usage.CacheMeasuredReadTokens != 70 { | |
| 47 | + | t.Fatalf("usage = %+v", usage) | |
| 48 | + | } | |
| 49 | + | } | |
| 50 | + | ||
| 51 | + | func TestDeepSeekRejectsIncompleteCacheUsage(t *testing.T) { | |
| 52 | + | hits := 70 | |
| 53 | + | usage := normalizeDeepSeekUsage(100, 9, &hits, nil) | |
| 54 | + | if usage.InputTokens != 100 || usage.CacheReadTokens != 0 || usage.CacheMeasuredPromptTokens != 0 { | |
| 55 | + | t.Fatalf("usage = %+v", usage) | |
| 56 | + | } | |
| 57 | + | } | |
| 58 | + | ||
| 31 | 59 | func TestOpenAINormalizesMeasuredCacheUsage(t *testing.T) { | |
| 32 | 60 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 33 | 61 | w.Header().Set("content-type", "text/event-stream") | |
| @@ -28,6 +28,34 @@ func TestOpenAIReportsReasoningTokens(t *testing.T) { | |||||
| 28 | } | 28 | } | ||
| 29 | } | 29 | } | ||
| 30 | 30 | ||||
| 31 | + | func TestDeepSeekNormalizesNativeCacheUsage(t *testing.T) { | |||
| 32 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |||
| 33 | + | w.Header().Set("content-type", "text/event-stream") | |||
| 34 | + | fmt.Fprintln(w, `data: {"choices":[{"delta":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":100,"prompt_cache_hit_tokens":70,"prompt_cache_miss_tokens":30,"completion_tokens":9}}`) | |||
| 35 | + | fmt.Fprintln(w) | |||
| 36 | + | fmt.Fprintln(w, "data: [DONE]") | |||
| 37 | + | fmt.Fprintln(w) | |||
| 38 | + | })) | |||
| 39 | + | defer srv.Close() | |||
| 40 | + | ||||
| 41 | + | events, err := NewDeepSeek("test", srv.URL).Stream(context.Background(), Request{Model: "deepseek-flash"}) | |||
| 42 | + | if err != nil { | |||
| 43 | + | t.Fatal(err) | |||
| 44 | + | } | |||
| 45 | + | usage := usageFromEvents(events) | |||
| 46 | + | if usage.InputTokens != 30 || usage.CacheReadTokens != 70 || usage.CacheWriteTokens != 0 || usage.CacheMeasuredPromptTokens != 100 || usage.CacheMeasuredReadTokens != 70 { | |||
| 47 | + | t.Fatalf("usage = %+v", usage) | |||
| 48 | + | } | |||
| 49 | + | } | |||
| 50 | + | ||||
| 51 | + | func TestDeepSeekRejectsIncompleteCacheUsage(t *testing.T) { | |||
| 52 | + | hits := 70 | |||
| 53 | + | usage := normalizeDeepSeekUsage(100, 9, &hits, nil) | |||
| 54 | + | if usage.InputTokens != 100 || usage.CacheReadTokens != 0 || usage.CacheMeasuredPromptTokens != 0 { | |||
| 55 | + | t.Fatalf("usage = %+v", usage) | |||
| 56 | + | } | |||
| 57 | + | } | |||
| 58 | + | ||||
| 31 | func TestOpenAINormalizesMeasuredCacheUsage(t *testing.T) { | 59 | func TestOpenAINormalizesMeasuredCacheUsage(t *testing.T) { | ||
| 32 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | 60 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| 33 | w.Header().Set("content-type", "text/event-stream") | 61 | w.Header().Set("content-type", "text/event-stream") | ||