Add print mode generation stats
- commit
- 6c955d6a61114aa18a4d3b7f89b71715cd5dfa96
- author
- patriceckhart · · 2026-08-03 05:25 UTC
- parent
- 2501d48
- signature
- unsigned
- refs
- tag: v0.3.31
22 files · +363 -30
M README.md +4 -0
| @@ -189,6 +189,7 @@ The first time you launch a newer zot binary, the TUI shows the GitHub release n | |||
| 189 | 189 | zot # interactive tui | |
| 190 | 190 | zot "fix the failing test" # tui, pre-filled prompt | |
| 191 | 191 | zot -p "list all go files" # print final text, exit | |
| 192 | + | zot -p --stats stats.json "task" # print final text and write generation stats | |
| 192 | 193 | zot --json "refactor main.go" # newline-delimited json events, exit | |
| 193 | 194 | zot --continue # resume the most recent session for this cwd | |
| 194 | 195 | zot --resume # pick a session to resume | |
| @@ -196,6 +197,8 @@ zot --list-models # show supported models | |||
| 196 | 197 | zot --help | |
| 197 | 198 | ``` | |
| 198 | 199 | ||
| 200 | + | Print-mode stats contain `provider`, `model`, `prompt_tokens`, `reasoning_tokens`, `generated_output_tokens`, and `elapsed_ms`. Counts cover all model turns triggered by the prompt, including tool loops. Prompt tokens include cache reads and writes. `reasoning_tokens` is `null` when the provider does not report a separate count; in that case `generated_output_tokens` is the provider's total output count and may include reasoning. Elapsed time covers the agent run, not startup and credential resolution. The file is written only after a successful run. | |
| 201 | + | ||
| 199 | 202 | ## Flags | |
| 200 | 203 | ||
| 201 | 204 | | Flag | Description | | |
| @@ -208,6 +211,7 @@ zot --help | |||
| 208 | 211 | | `--system-prompt <text>` | Replace the default system prompt for this run (also overrides `$ZOT_HOME/SYSTEM.md`). | | |
| 209 | 212 | | `--append-system-prompt <text>` | Append text to the system prompt (repeatable). | | |
| 210 | 213 | | `--reasoning off\|minimum\|low\|medium\|high\|xhigh\|max` | Set thinking level on supported models (default: off). `max` is a separate opt-in tier above `xhigh`. | | |
| 214 | + | | `--stats <path>` | With `-p`/`--print`, write generation statistics as JSON. | | |
| 211 | 215 | | `-c`, `--continue` | Resume the latest session for this cwd. | | |
| 212 | 216 | | `-r`, `--resume` | Pick a session to resume. | | |
| 213 | 217 | | `--session <path>` | Resume a specific session file. | | |
| @@ -189,6 +189,7 @@ The first time you launch a newer zot binary, the TUI shows the GitHub release n | |||||
| 189 | zot # interactive tui | 189 | zot # interactive tui | ||
| 190 | zot "fix the failing test" # tui, pre-filled prompt | 190 | zot "fix the failing test" # tui, pre-filled prompt | ||
| 191 | zot -p "list all go files" # print final text, exit | 191 | zot -p "list all go files" # print final text, exit | ||
| 192 | + | zot -p --stats stats.json "task" # print final text and write generation stats | |||
| 192 | zot --json "refactor main.go" # newline-delimited json events, exit | 193 | zot --json "refactor main.go" # newline-delimited json events, exit | ||
| 193 | zot --continue # resume the most recent session for this cwd | 194 | zot --continue # resume the most recent session for this cwd | ||
| 194 | zot --resume # pick a session to resume | 195 | zot --resume # pick a session to resume | ||
| @@ -196,6 +197,8 @@ zot --list-models # show supported models | |||||
| 196 | zot --help | 197 | zot --help | ||
| 197 | ``` | 198 | ``` | ||
| 198 | 199 | ||||
| 200 | + | Print-mode stats contain `provider`, `model`, `prompt_tokens`, `reasoning_tokens`, `generated_output_tokens`, and `elapsed_ms`. Counts cover all model turns triggered by the prompt, including tool loops. Prompt tokens include cache reads and writes. `reasoning_tokens` is `null` when the provider does not report a separate count; in that case `generated_output_tokens` is the provider's total output count and may include reasoning. Elapsed time covers the agent run, not startup and credential resolution. The file is written only after a successful run. | |||
| 201 | + | ||||
| 199 | ## Flags | 202 | ## Flags | ||
| 200 | 203 | ||||
| 201 | | Flag | Description | | 204 | | Flag | Description | | ||
| @@ -208,6 +211,7 @@ zot --help | |||||
| 208 | | `--system-prompt <text>` | Replace the default system prompt for this run (also overrides `$ZOT_HOME/SYSTEM.md`). | | 211 | | `--system-prompt <text>` | Replace the default system prompt for this run (also overrides `$ZOT_HOME/SYSTEM.md`). | | ||
| 209 | | `--append-system-prompt <text>` | Append text to the system prompt (repeatable). | | 212 | | `--append-system-prompt <text>` | Append text to the system prompt (repeatable). | | ||
| 210 | | `--reasoning off\|minimum\|low\|medium\|high\|xhigh\|max` | Set thinking level on supported models (default: off). `max` is a separate opt-in tier above `xhigh`. | | 213 | | `--reasoning off\|minimum\|low\|medium\|high\|xhigh\|max` | Set thinking level on supported models (default: off). `max` is a separate opt-in tier above `xhigh`. | | ||
| 214 | + | | `--stats <path>` | With `-p`/`--print`, write generation statistics as JSON. | | |||
| 211 | | `-c`, `--continue` | Resume the latest session for this cwd. | | 215 | | `-c`, `--continue` | Resume the latest session for this cwd. | | ||
| 212 | | `-r`, `--resume` | Pick a session to resume. | | 216 | | `-r`, `--resume` | Pick a session to resume. | | ||
| 213 | | `--session <path>` | Resume a specific session file. | | 217 | | `--session <path>` | Resume a specific session file. | | ||
M docs/rpc.md +2 -2
| @@ -121,7 +121,7 @@ Response data: | |||
| 121 | 121 | "cwd": "/Users/pat/Developer/zot", | |
| 122 | 122 | "message_count": 12, | |
| 123 | 123 | "busy": false, | |
| 124 | - | "usage": {"input": 1234, "output": 567, "cache_read": 890, "cache_write": 0, "cost_usd": 0.0123} | |
| 124 | + | "usage": {"input": 1234, "output": 567, "reasoning": 123, "cache_read": 890, "cache_write": 0, "cost_usd": 0.0123} | |
| 125 | 125 | } | |
| 126 | 126 | ``` | |
| 127 | 127 | ||
| @@ -197,7 +197,7 @@ Stream notifications during a `prompt` or `compact`. None carry an `id`. | |||
| 197 | 197 | | `tool_progress` | `id`, `text` | Optional progress line from the tool while it runs | | |
| 198 | 198 | | `tool_result` | `id`, `is_error`, `content` | Tool finished | | |
| 199 | 199 | | `assistant_message` | `content`, `time` | Final assistant message after the model turn ends | | |
| 200 | - | | `usage` | `input`, `output`, `cache_read`, `cache_write`, `cost_usd`, `cumulative` | Per-turn + cumulative tokens / cost | | |
| 200 | + | | `usage` | `input`, `output`, `reasoning`, `cache_read`, `cache_write`, `cost_usd`, `cumulative` | Per-turn + cumulative tokens / cost. `reasoning` is null when unavailable. | | |
| 201 | 201 | | `turn_end` | `stop`, optional `error` | One model call finished. `stop` is `end_turn`, `tool_use`, `length`, `error`, or `aborted` | | |
| 202 | 202 | | `done` | (none) | The whole prompt/compact completed (success or error) | | |
| 203 | 203 | | `error` | `message` | Non-fatal error message | | |
| @@ -121,7 +121,7 @@ Response data: | |||||
| 121 | "cwd": "/Users/pat/Developer/zot", | 121 | "cwd": "/Users/pat/Developer/zot", | ||
| 122 | "message_count": 12, | 122 | "message_count": 12, | ||
| 123 | "busy": false, | 123 | "busy": false, | ||
| 124 | - | "usage": {"input": 1234, "output": 567, "cache_read": 890, "cache_write": 0, "cost_usd": 0.0123} | 124 | + | "usage": {"input": 1234, "output": 567, "reasoning": 123, "cache_read": 890, "cache_write": 0, "cost_usd": 0.0123} |
| 125 | } | 125 | } | ||
| 126 | ``` | 126 | ``` | ||
| 127 | 127 | ||||
| @@ -197,7 +197,7 @@ Stream notifications during a `prompt` or `compact`. None carry an `id`. | |||||
| 197 | | `tool_progress` | `id`, `text` | Optional progress line from the tool while it runs | | 197 | | `tool_progress` | `id`, `text` | Optional progress line from the tool while it runs | | ||
| 198 | | `tool_result` | `id`, `is_error`, `content` | Tool finished | | 198 | | `tool_result` | `id`, `is_error`, `content` | Tool finished | | ||
| 199 | | `assistant_message` | `content`, `time` | Final assistant message after the model turn ends | | 199 | | `assistant_message` | `content`, `time` | Final assistant message after the model turn ends | | ||
| 200 | - | | `usage` | `input`, `output`, `cache_read`, `cache_write`, `cost_usd`, `cumulative` | Per-turn + cumulative tokens / cost | | 200 | + | | `usage` | `input`, `output`, `reasoning`, `cache_read`, `cache_write`, `cost_usd`, `cumulative` | Per-turn + cumulative tokens / cost. `reasoning` is null when unavailable. | |
| 201 | | `turn_end` | `stop`, optional `error` | One model call finished. `stop` is `end_turn`, `tool_use`, `length`, `error`, or `aborted` | | 201 | | `turn_end` | `stop`, optional `error` | One model call finished. `stop` is `end_turn`, `tool_use`, `length`, `error`, or `aborted` | | ||
| 202 | | `done` | (none) | The whole prompt/compact completed (success or error) | | 202 | | `done` | (none) | The whole prompt/compact completed (success or error) | | ||
| 203 | | `error` | `message` | Non-fatal error message | | 203 | | `error` | `message` | Non-fatal error message | | ||
M packages/agent/args.go +11 -0
| @@ -106,6 +106,7 @@ type Args struct { | |||
| 106 | 106 | ListModels bool | |
| 107 | 107 | Help bool | |
| 108 | 108 | Version bool | |
| 109 | + | StatsPath string // write print-mode generation statistics as JSON | |
| 109 | 110 | ||
| 110 | 111 | Prompt string // concatenated positional args | |
| 111 | 112 | ||
| @@ -247,6 +248,12 @@ func ParseArgs(in []string) (Args, error) { | |||
| 247 | 248 | } | |
| 248 | 249 | t := float32(f) | |
| 249 | 250 | a.Temperature = &t | |
| 251 | + | case "--stats": | |
| 252 | + | v, err := want(&i, arg) | |
| 253 | + | if err != nil { | |
| 254 | + | return a, err | |
| 255 | + | } | |
| 256 | + | a.StatsPath = v | |
| 250 | 257 | case "--session": | |
| 251 | 258 | v, err := want(&i, arg) | |
| 252 | 259 | if err != nil { | |
| @@ -299,6 +306,9 @@ func ParseArgs(in []string) (Args, error) { | |||
| 299 | 306 | a.Prompt = strings.Join(positional, " ") | |
| 300 | 307 | } | |
| 301 | 308 | ||
| 309 | + | if a.StatsPath != "" && a.Mode != ModePrint { | |
| 310 | + | return a, fmt.Errorf("--stats requires -p or --print") | |
| 311 | + | } | |
| 302 | 312 | if a.CWD == "" { | |
| 303 | 313 | a.CWD, _ = os.Getwd() | |
| 304 | 314 | } | |
| @@ -412,6 +422,7 @@ func PrintHelp(version string) { | |||
| 412 | 422 | row{"--temperature N", "sampling temperature, 0 to 2 (omit for provider default)"}, | |
| 413 | 423 | ) | |
| 414 | 424 | section("prompt and session flags", | |
| 425 | + | row{"--stats PATH", "write print-mode generation stats as json"}, | |
| 415 | 426 | row{"--system-prompt TEXT", "replace the default system prompt"}, | |
| 416 | 427 | row{"--append-system-prompt TEXT", "append to the system prompt (repeatable)"}, | |
| 417 | 428 | row{"-c, --continue", "continue the most recent session for this cwd"}, | |
| @@ -106,6 +106,7 @@ type Args struct { | |||||
| 106 | ListModels bool | 106 | ListModels bool | ||
| 107 | Help bool | 107 | Help bool | ||
| 108 | Version bool | 108 | Version bool | ||
| 109 | + | StatsPath string // write print-mode generation statistics as JSON | |||
| 109 | 110 | ||||
| 110 | Prompt string // concatenated positional args | 111 | Prompt string // concatenated positional args | ||
| 111 | 112 | ||||
| @@ -247,6 +248,12 @@ func ParseArgs(in []string) (Args, error) { | |||||
| 247 | } | 248 | } | ||
| 248 | t := float32(f) | 249 | t := float32(f) | ||
| 249 | a.Temperature = &t | 250 | a.Temperature = &t | ||
| 251 | + | case "--stats": | |||
| 252 | + | v, err := want(&i, arg) | |||
| 253 | + | if err != nil { | |||
| 254 | + | return a, err | |||
| 255 | + | } | |||
| 256 | + | a.StatsPath = v | |||
| 250 | case "--session": | 257 | case "--session": | ||
| 251 | v, err := want(&i, arg) | 258 | v, err := want(&i, arg) | ||
| 252 | if err != nil { | 259 | if err != nil { | ||
| @@ -299,6 +306,9 @@ func ParseArgs(in []string) (Args, error) { | |||||
| 299 | a.Prompt = strings.Join(positional, " ") | 306 | a.Prompt = strings.Join(positional, " ") | ||
| 300 | } | 307 | } | ||
| 301 | 308 | ||||
| 309 | + | if a.StatsPath != "" && a.Mode != ModePrint { | |||
| 310 | + | return a, fmt.Errorf("--stats requires -p or --print") | |||
| 311 | + | } | |||
| 302 | if a.CWD == "" { | 312 | if a.CWD == "" { | ||
| 303 | a.CWD, _ = os.Getwd() | 313 | a.CWD, _ = os.Getwd() | ||
| 304 | } | 314 | } | ||
| @@ -412,6 +422,7 @@ func PrintHelp(version string) { | |||||
| 412 | row{"--temperature N", "sampling temperature, 0 to 2 (omit for provider default)"}, | 422 | row{"--temperature N", "sampling temperature, 0 to 2 (omit for provider default)"}, | ||
| 413 | ) | 423 | ) | ||
| 414 | section("prompt and session flags", | 424 | section("prompt and session flags", | ||
| 425 | + | row{"--stats PATH", "write print-mode generation stats as json"}, | |||
| 415 | row{"--system-prompt TEXT", "replace the default system prompt"}, | 426 | row{"--system-prompt TEXT", "replace the default system prompt"}, | ||
| 416 | row{"--append-system-prompt TEXT", "append to the system prompt (repeatable)"}, | 427 | row{"--append-system-prompt TEXT", "append to the system prompt (repeatable)"}, | ||
| 417 | row{"-c, --continue", "continue the most recent session for this cwd"}, | 428 | row{"-c, --continue", "continue the most recent session for this cwd"}, | ||
M packages/agent/args_test.go +14 -0
| @@ -33,6 +33,20 @@ func TestParseArgsYes(t *testing.T) { | |||
| 33 | 33 | } | |
| 34 | 34 | } | |
| 35 | 35 | ||
| 36 | + | func TestParseArgsStatsRequiresPrintMode(t *testing.T) { | |
| 37 | + | args, err := ParseArgs([]string{"-p", "--stats", "stats.json", "hi"}) | |
| 38 | + | if err != nil { | |
| 39 | + | t.Fatal(err) | |
| 40 | + | } | |
| 41 | + | if args.StatsPath != "stats.json" || args.Mode != ModePrint { | |
| 42 | + | t.Fatalf("StatsPath=%q Mode=%q", args.StatsPath, args.Mode) | |
| 43 | + | } | |
| 44 | + | ||
| 45 | + | if _, err := ParseArgs([]string{"--stats", "stats.json", "hi"}); err == nil { | |
| 46 | + | t.Fatal("ParseArgs accepted --stats without print mode") | |
| 47 | + | } | |
| 48 | + | } | |
| 49 | + | ||
| 36 | 50 | func TestParseArgsStream(t *testing.T) { | |
| 37 | 51 | args, err := ParseArgs([]string{"--stream", "hi"}) | |
| 38 | 52 | if err != nil { | |
| @@ -33,6 +33,20 @@ func TestParseArgsYes(t *testing.T) { | |||||
| 33 | } | 33 | } | ||
| 34 | } | 34 | } | ||
| 35 | 35 | ||||
| 36 | + | func TestParseArgsStatsRequiresPrintMode(t *testing.T) { | |||
| 37 | + | args, err := ParseArgs([]string{"-p", "--stats", "stats.json", "hi"}) | |||
| 38 | + | if err != nil { | |||
| 39 | + | t.Fatal(err) | |||
| 40 | + | } | |||
| 41 | + | if args.StatsPath != "stats.json" || args.Mode != ModePrint { | |||
| 42 | + | t.Fatalf("StatsPath=%q Mode=%q", args.StatsPath, args.Mode) | |||
| 43 | + | } | |||
| 44 | + | ||||
| 45 | + | if _, err := ParseArgs([]string{"--stats", "stats.json", "hi"}); err == nil { | |||
| 46 | + | t.Fatal("ParseArgs accepted --stats without print mode") | |||
| 47 | + | } | |||
| 48 | + | } | |||
| 49 | + | ||||
| 36 | func TestParseArgsStream(t *testing.T) { | 50 | func TestParseArgsStream(t *testing.T) { | ||
| 37 | args, err := ParseArgs([]string{"--stream", "hi"}) | 51 | args, err := ParseArgs([]string{"--stream", "hi"}) | ||
| 38 | if err != nil { | 52 | if err != nil { | ||
M packages/agent/cli.go +49 -2
| @@ -330,6 +330,45 @@ func wireNonInteractiveAgentExtHooks(ctx context.Context, ag *core.Agent, extMgr | |||
| 330 | 330 | ag.OnEvent = func(ev core.AgentEvent) { fanoutAgentEvent(extMgr, ev) } | |
| 331 | 331 | } | |
| 332 | 332 | ||
| 333 | + | type printStats struct { | |
| 334 | + | Provider string `json:"provider"` | |
| 335 | + | Model string `json:"model"` | |
| 336 | + | PromptTokens int `json:"prompt_tokens"` | |
| 337 | + | ReasoningTokens *int `json:"reasoning_tokens"` | |
| 338 | + | GeneratedOutputTokens int `json:"generated_output_tokens"` | |
| 339 | + | ElapsedMS int64 `json:"elapsed_ms"` | |
| 340 | + | } | |
| 341 | + | ||
| 342 | + | func writePrintStats(path, providerID, model string, usage provider.Usage, elapsed time.Duration) error { | |
| 343 | + | var reasoning *int | |
| 344 | + | generated := usage.OutputTokens | |
| 345 | + | if usage.ReasoningTokensKnown { | |
| 346 | + | count := usage.ReasoningTokens | |
| 347 | + | reasoning = &count | |
| 348 | + | generated -= count | |
| 349 | + | if generated < 0 { | |
| 350 | + | generated = 0 | |
| 351 | + | } | |
| 352 | + | } | |
| 353 | + | stats := printStats{ | |
| 354 | + | Provider: providerID, | |
| 355 | + | Model: model, | |
| 356 | + | PromptTokens: usage.InputTokens + usage.CacheReadTokens + usage.CacheWriteTokens, | |
| 357 | + | ReasoningTokens: reasoning, | |
| 358 | + | GeneratedOutputTokens: generated, | |
| 359 | + | ElapsedMS: elapsed.Milliseconds(), | |
| 360 | + | } | |
| 361 | + | data, err := json.MarshalIndent(stats, "", " ") | |
| 362 | + | if err != nil { | |
| 363 | + | return fmt.Errorf("encode stats: %w", err) | |
| 364 | + | } | |
| 365 | + | data = append(data, '\n') | |
| 366 | + | if err := os.WriteFile(path, data, 0o644); err != nil { | |
| 367 | + | return fmt.Errorf("write stats %q: %w", path, err) | |
| 368 | + | } | |
| 369 | + | return nil | |
| 370 | + | } | |
| 371 | + | ||
| 333 | 372 | func runPrintMode(ctx context.Context, args Args, version string) error { | |
| 334 | 373 | if args.NoYolo { | |
| 335 | 374 | fmt.Fprintln(os.Stderr, "warning: --no-yolo has no effect in print mode (no interactive prompt available); tools will run without confirmation") | |
| @@ -360,9 +399,17 @@ func runPrintMode(ctx context.Context, args Args, version string) error { | |||
| 360 | 399 | return err | |
| 361 | 400 | } | |
| 362 | 401 | reloadResourcesAfterStartupPre(ctx, args, extMgr, r.Sandbox, ag) | |
| 363 | - | err = modes.RunPrint(ctx, ag, prompt, nil, os.Stdout) | |
| 402 | + | started := time.Now() | |
| 403 | + | usage, err := modes.RunPrint(ctx, ag, prompt, nil, os.Stdout) | |
| 404 | + | elapsed := time.Since(started) | |
| 364 | 405 | WriteNewTranscript(ag, sess, start) | |
| 365 | - | return err | |
| 406 | + | if err != nil { | |
| 407 | + | return err | |
| 408 | + | } | |
| 409 | + | if args.StatsPath != "" { | |
| 410 | + | return writePrintStats(args.StatsPath, r.Provider, r.Model, usage, elapsed) | |
| 411 | + | } | |
| 412 | + | return nil | |
| 366 | 413 | } | |
| 367 | 414 | ||
| 368 | 415 | func runStreamMode(ctx context.Context, args Args, version string) error { | |
| @@ -330,6 +330,45 @@ func wireNonInteractiveAgentExtHooks(ctx context.Context, ag *core.Agent, extMgr | |||||
| 330 | ag.OnEvent = func(ev core.AgentEvent) { fanoutAgentEvent(extMgr, ev) } | 330 | ag.OnEvent = func(ev core.AgentEvent) { fanoutAgentEvent(extMgr, ev) } | ||
| 331 | } | 331 | } | ||
| 332 | 332 | ||||
| 333 | + | type printStats struct { | |||
| 334 | + | Provider string `json:"provider"` | |||
| 335 | + | Model string `json:"model"` | |||
| 336 | + | PromptTokens int `json:"prompt_tokens"` | |||
| 337 | + | ReasoningTokens *int `json:"reasoning_tokens"` | |||
| 338 | + | GeneratedOutputTokens int `json:"generated_output_tokens"` | |||
| 339 | + | ElapsedMS int64 `json:"elapsed_ms"` | |||
| 340 | + | } | |||
| 341 | + | ||||
| 342 | + | func writePrintStats(path, providerID, model string, usage provider.Usage, elapsed time.Duration) error { | |||
| 343 | + | var reasoning *int | |||
| 344 | + | generated := usage.OutputTokens | |||
| 345 | + | if usage.ReasoningTokensKnown { | |||
| 346 | + | count := usage.ReasoningTokens | |||
| 347 | + | reasoning = &count | |||
| 348 | + | generated -= count | |||
| 349 | + | if generated < 0 { | |||
| 350 | + | generated = 0 | |||
| 351 | + | } | |||
| 352 | + | } | |||
| 353 | + | stats := printStats{ | |||
| 354 | + | Provider: providerID, | |||
| 355 | + | Model: model, | |||
| 356 | + | PromptTokens: usage.InputTokens + usage.CacheReadTokens + usage.CacheWriteTokens, | |||
| 357 | + | ReasoningTokens: reasoning, | |||
| 358 | + | GeneratedOutputTokens: generated, | |||
| 359 | + | ElapsedMS: elapsed.Milliseconds(), | |||
| 360 | + | } | |||
| 361 | + | data, err := json.MarshalIndent(stats, "", " ") | |||
| 362 | + | if err != nil { | |||
| 363 | + | return fmt.Errorf("encode stats: %w", err) | |||
| 364 | + | } | |||
| 365 | + | data = append(data, '\n') | |||
| 366 | + | if err := os.WriteFile(path, data, 0o644); err != nil { | |||
| 367 | + | return fmt.Errorf("write stats %q: %w", path, err) | |||
| 368 | + | } | |||
| 369 | + | return nil | |||
| 370 | + | } | |||
| 371 | + | ||||
| 333 | func runPrintMode(ctx context.Context, args Args, version string) error { | 372 | func runPrintMode(ctx context.Context, args Args, version string) error { | ||
| 334 | if args.NoYolo { | 373 | if args.NoYolo { | ||
| 335 | fmt.Fprintln(os.Stderr, "warning: --no-yolo has no effect in print mode (no interactive prompt available); tools will run without confirmation") | 374 | fmt.Fprintln(os.Stderr, "warning: --no-yolo has no effect in print mode (no interactive prompt available); tools will run without confirmation") | ||
| @@ -360,9 +399,17 @@ func runPrintMode(ctx context.Context, args Args, version string) error { | |||||
| 360 | return err | 399 | return err | ||
| 361 | } | 400 | } | ||
| 362 | reloadResourcesAfterStartupPre(ctx, args, extMgr, r.Sandbox, ag) | 401 | reloadResourcesAfterStartupPre(ctx, args, extMgr, r.Sandbox, ag) | ||
| 363 | - | err = modes.RunPrint(ctx, ag, prompt, nil, os.Stdout) | 402 | + | started := time.Now() |
| 403 | + | usage, err := modes.RunPrint(ctx, ag, prompt, nil, os.Stdout) | |||
| 404 | + | elapsed := time.Since(started) | |||
| 364 | WriteNewTranscript(ag, sess, start) | 405 | WriteNewTranscript(ag, sess, start) | ||
| 365 | - | return err | 406 | + | if err != nil { |
| 407 | + | return err | |||
| 408 | + | } | |||
| 409 | + | if args.StatsPath != "" { | |||
| 410 | + | return writePrintStats(args.StatsPath, r.Provider, r.Model, usage, elapsed) | |||
| 411 | + | } | |||
| 412 | + | return nil | |||
| 366 | } | 413 | } | ||
| 367 | 414 | ||||
| 368 | func runStreamMode(ctx context.Context, args Args, version string) error { | 415 | func runStreamMode(ctx context.Context, args Args, version string) error { | ||
M packages/agent/modes/json.go +9 -0
| @@ -73,12 +73,14 @@ func EventToJSON(ev core.AgentEvent) map[string]any { | |||
| 73 | 73 | case core.EvUsage: | |
| 74 | 74 | m["input"] = e.Usage.InputTokens | |
| 75 | 75 | m["output"] = e.Usage.OutputTokens | |
| 76 | + | m["reasoning"] = nullableReasoningTokens(e.Usage) | |
| 76 | 77 | m["cache_read"] = e.Usage.CacheReadTokens | |
| 77 | 78 | m["cache_write"] = e.Usage.CacheWriteTokens | |
| 78 | 79 | m["cost_usd"] = e.Usage.CostUSD | |
| 79 | 80 | m["cumulative"] = map[string]any{ | |
| 80 | 81 | "input": e.Cumulative.InputTokens, | |
| 81 | 82 | "output": e.Cumulative.OutputTokens, | |
| 83 | + | "reasoning": nullableReasoningTokens(e.Cumulative), | |
| 82 | 84 | "cache_read": e.Cumulative.CacheReadTokens, | |
| 83 | 85 | "cache_write": e.Cumulative.CacheWriteTokens, | |
| 84 | 86 | "cost_usd": e.Cumulative.CostUSD, | |
| @@ -119,6 +121,13 @@ func ContentToJSON(blocks []provider.Content) []map[string]any { | |||
| 119 | 121 | return out | |
| 120 | 122 | } | |
| 121 | 123 | ||
| 124 | + | func nullableReasoningTokens(usage provider.Usage) any { | |
| 125 | + | if !usage.ReasoningTokensKnown { | |
| 126 | + | return nil | |
| 127 | + | } | |
| 128 | + | return usage.ReasoningTokens | |
| 129 | + | } | |
| 130 | + | ||
| 122 | 131 | func jsonString(s string) string { | |
| 123 | 132 | b, _ := json.Marshal(s) | |
| 124 | 133 | return string(b) | |
| @@ -73,12 +73,14 @@ func EventToJSON(ev core.AgentEvent) map[string]any { | |||||
| 73 | case core.EvUsage: | 73 | case core.EvUsage: | ||
| 74 | m["input"] = e.Usage.InputTokens | 74 | m["input"] = e.Usage.InputTokens | ||
| 75 | m["output"] = e.Usage.OutputTokens | 75 | m["output"] = e.Usage.OutputTokens | ||
| 76 | + | m["reasoning"] = nullableReasoningTokens(e.Usage) | |||
| 76 | m["cache_read"] = e.Usage.CacheReadTokens | 77 | m["cache_read"] = e.Usage.CacheReadTokens | ||
| 77 | m["cache_write"] = e.Usage.CacheWriteTokens | 78 | m["cache_write"] = e.Usage.CacheWriteTokens | ||
| 78 | m["cost_usd"] = e.Usage.CostUSD | 79 | m["cost_usd"] = e.Usage.CostUSD | ||
| 79 | m["cumulative"] = map[string]any{ | 80 | m["cumulative"] = map[string]any{ | ||
| 80 | "input": e.Cumulative.InputTokens, | 81 | "input": e.Cumulative.InputTokens, | ||
| 81 | "output": e.Cumulative.OutputTokens, | 82 | "output": e.Cumulative.OutputTokens, | ||
| 83 | + | "reasoning": nullableReasoningTokens(e.Cumulative), | |||
| 82 | "cache_read": e.Cumulative.CacheReadTokens, | 84 | "cache_read": e.Cumulative.CacheReadTokens, | ||
| 83 | "cache_write": e.Cumulative.CacheWriteTokens, | 85 | "cache_write": e.Cumulative.CacheWriteTokens, | ||
| 84 | "cost_usd": e.Cumulative.CostUSD, | 86 | "cost_usd": e.Cumulative.CostUSD, | ||
| @@ -119,6 +121,13 @@ func ContentToJSON(blocks []provider.Content) []map[string]any { | |||||
| 119 | return out | 121 | return out | ||
| 120 | } | 122 | } | ||
| 121 | 123 | ||||
| 124 | + | func nullableReasoningTokens(usage provider.Usage) any { | |||
| 125 | + | if !usage.ReasoningTokensKnown { | |||
| 126 | + | return nil | |||
| 127 | + | } | |||
| 128 | + | return usage.ReasoningTokens | |||
| 129 | + | } | |||
| 130 | + | ||||
| 122 | func jsonString(s string) string { | 131 | func jsonString(s string) string { | ||
| 123 | b, _ := json.Marshal(s) | 132 | b, _ := json.Marshal(s) | ||
| 124 | return string(b) | 133 | return string(b) | ||
M packages/agent/modes/print.go +15 -5
| @@ -12,10 +12,13 @@ import ( | |||
| 12 | 12 | ) | |
| 13 | 13 | ||
| 14 | 14 | // RunPrint runs the agent to completion and writes only the final | |
| 15 | - | // assistant text block to out. Exit code comes from the caller. | |
| 16 | - | func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provider.ImageBlock, out io.Writer) error { | |
| 15 | + | // assistant text block to out. It returns usage for this invocation, | |
| 16 | + | // excluding any cumulative usage restored from a session. | |
| 17 | + | func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provider.ImageBlock, out io.Writer) (provider.Usage, error) { | |
| 17 | 18 | var finalText strings.Builder | |
| 18 | 19 | var lastAssistant string | |
| 20 | + | var usage provider.Usage | |
| 21 | + | var haveUsage bool | |
| 19 | 22 | var runErr error | |
| 20 | 23 | ||
| 21 | 24 | sink := func(ev core.AgentEvent) { | |
| @@ -34,6 +37,13 @@ func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provi | |||
| 34 | 37 | if sb.Len() > 0 { | |
| 35 | 38 | lastAssistant = sb.String() | |
| 36 | 39 | } | |
| 40 | + | case core.EvUsage: | |
| 41 | + | if haveUsage { | |
| 42 | + | usage = usage.Add(e.Usage) | |
| 43 | + | } else { | |
| 44 | + | usage = e.Usage | |
| 45 | + | haveUsage = true | |
| 46 | + | } | |
| 37 | 47 | case core.EvTurnEnd: | |
| 38 | 48 | if e.Err != nil { | |
| 39 | 49 | runErr = e.Err | |
| @@ -42,10 +52,10 @@ func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provi | |||
| 42 | 52 | } | |
| 43 | 53 | ||
| 44 | 54 | if err := ag.Prompt(ctx, prompt, images, sink); err != nil { | |
| 45 | - | return err | |
| 55 | + | return usage, err | |
| 46 | 56 | } | |
| 47 | 57 | if runErr != nil { | |
| 48 | - | return runErr | |
| 58 | + | return usage, runErr | |
| 49 | 59 | } | |
| 50 | 60 | ||
| 51 | 61 | finalText.WriteString(lastAssistant) | |
| @@ -53,5 +63,5 @@ func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provi | |||
| 53 | 63 | finalText.WriteString("\n") | |
| 54 | 64 | } | |
| 55 | 65 | _, err := fmt.Fprint(out, finalText.String()) | |
| 56 | - | return err | |
| 66 | + | return usage, err | |
| 57 | 67 | } | |
| @@ -12,10 +12,13 @@ import ( | |||||
| 12 | ) | 12 | ) | ||
| 13 | 13 | ||||
| 14 | // RunPrint runs the agent to completion and writes only the final | 14 | // RunPrint runs the agent to completion and writes only the final | ||
| 15 | - | // assistant text block to out. Exit code comes from the caller. | 15 | + | // assistant text block to out. It returns usage for this invocation, |
| 16 | - | func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provider.ImageBlock, out io.Writer) error { | 16 | + | // excluding any cumulative usage restored from a session. |
| 17 | + | func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provider.ImageBlock, out io.Writer) (provider.Usage, error) { | |||
| 17 | var finalText strings.Builder | 18 | var finalText strings.Builder | ||
| 18 | var lastAssistant string | 19 | var lastAssistant string | ||
| 20 | + | var usage provider.Usage | |||
| 21 | + | var haveUsage bool | |||
| 19 | var runErr error | 22 | var runErr error | ||
| 20 | 23 | ||||
| 21 | sink := func(ev core.AgentEvent) { | 24 | sink := func(ev core.AgentEvent) { | ||
| @@ -34,6 +37,13 @@ func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provi | |||||
| 34 | if sb.Len() > 0 { | 37 | if sb.Len() > 0 { | ||
| 35 | lastAssistant = sb.String() | 38 | lastAssistant = sb.String() | ||
| 36 | } | 39 | } | ||
| 40 | + | case core.EvUsage: | |||
| 41 | + | if haveUsage { | |||
| 42 | + | usage = usage.Add(e.Usage) | |||
| 43 | + | } else { | |||
| 44 | + | usage = e.Usage | |||
| 45 | + | haveUsage = true | |||
| 46 | + | } | |||
| 37 | case core.EvTurnEnd: | 47 | case core.EvTurnEnd: | ||
| 38 | if e.Err != nil { | 48 | if e.Err != nil { | ||
| 39 | runErr = e.Err | 49 | runErr = e.Err | ||
| @@ -42,10 +52,10 @@ func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provi | |||||
| 42 | } | 52 | } | ||
| 43 | 53 | ||||
| 44 | if err := ag.Prompt(ctx, prompt, images, sink); err != nil { | 54 | if err := ag.Prompt(ctx, prompt, images, sink); err != nil { | ||
| 45 | - | return err | 55 | + | return usage, err |
| 46 | } | 56 | } | ||
| 47 | if runErr != nil { | 57 | if runErr != nil { | ||
| 48 | - | return runErr | 58 | + | return usage, runErr |
| 49 | } | 59 | } | ||
| 50 | 60 | ||||
| 51 | finalText.WriteString(lastAssistant) | 61 | finalText.WriteString(lastAssistant) | ||
| @@ -53,5 +63,5 @@ func RunPrint(ctx context.Context, ag *core.Agent, prompt string, images []provi | |||||
| 53 | finalText.WriteString("\n") | 63 | finalText.WriteString("\n") | ||
| 54 | } | 64 | } | ||
| 55 | _, err := fmt.Fprint(out, finalText.String()) | 65 | _, err := fmt.Fprint(out, finalText.String()) | ||
| 56 | - | return err | 66 | + | return usage, err |
| 57 | } | 67 | } | ||
A packages/agent/modes/print_test.go +50 -0
| @@ -0,0 +1,50 @@ | |||
| 1 | + | package modes | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "bytes" | |
| 5 | + | "context" | |
| 6 | + | "testing" | |
| 7 | + | ||
| 8 | + | "github.com/patriceckhart/zot/packages/core" | |
| 9 | + | "github.com/patriceckhart/zot/packages/provider" | |
| 10 | + | ) | |
| 11 | + | ||
| 12 | + | type printTestClient struct{} | |
| 13 | + | ||
| 14 | + | func (printTestClient) Name() string { return "test" } | |
| 15 | + | ||
| 16 | + | func (printTestClient) Stream(context.Context, provider.Request) (<-chan provider.Event, error) { | |
| 17 | + | events := make(chan provider.Event, 3) | |
| 18 | + | events <- provider.EventStart{Model: "test-model", Provider: "test"} | |
| 19 | + | events <- provider.EventUsage{Usage: provider.Usage{ | |
| 20 | + | InputTokens: 12, | |
| 21 | + | OutputTokens: 9, | |
| 22 | + | ReasoningTokens: 6, | |
| 23 | + | ReasoningTokensKnown: true, | |
| 24 | + | }} | |
| 25 | + | events <- provider.EventDone{ | |
| 26 | + | Stop: provider.StopEnd, | |
| 27 | + | Message: provider.Message{Role: provider.RoleAssistant, Content: []provider.Content{ | |
| 28 | + | provider.TextBlock{Text: "answer"}, | |
| 29 | + | }}, | |
| 30 | + | } | |
| 31 | + | close(events) | |
| 32 | + | return events, nil | |
| 33 | + | } | |
| 34 | + | ||
| 35 | + | func TestRunPrintReturnsInvocationUsage(t *testing.T) { | |
| 36 | + | agent := core.NewAgent(printTestClient{}, "test-model", "", core.Registry{}) | |
| 37 | + | agent.SeedCost(provider.Usage{InputTokens: 100, OutputTokens: 50}) | |
| 38 | + | var output bytes.Buffer | |
| 39 | + | ||
| 40 | + | usage, err := RunPrint(context.Background(), agent, "question", nil, &output) | |
| 41 | + | if err != nil { | |
| 42 | + | t.Fatal(err) | |
| 43 | + | } | |
| 44 | + | if output.String() != "answer\n" { | |
| 45 | + | t.Fatalf("output = %q", output.String()) | |
| 46 | + | } | |
| 47 | + | if usage.InputTokens != 12 || usage.OutputTokens != 9 || usage.ReasoningTokens != 6 || !usage.ReasoningTokensKnown { | |
| 48 | + | t.Fatalf("usage = %+v", usage) | |
| 49 | + | } | |
| 50 | + | } | |
| @@ -0,0 +1,50 @@ | |||||
| 1 | + | package modes | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "bytes" | |||
| 5 | + | "context" | |||
| 6 | + | "testing" | |||
| 7 | + | ||||
| 8 | + | "github.com/patriceckhart/zot/packages/core" | |||
| 9 | + | "github.com/patriceckhart/zot/packages/provider" | |||
| 10 | + | ) | |||
| 11 | + | ||||
| 12 | + | type printTestClient struct{} | |||
| 13 | + | ||||
| 14 | + | func (printTestClient) Name() string { return "test" } | |||
| 15 | + | ||||
| 16 | + | func (printTestClient) Stream(context.Context, provider.Request) (<-chan provider.Event, error) { | |||
| 17 | + | events := make(chan provider.Event, 3) | |||
| 18 | + | events <- provider.EventStart{Model: "test-model", Provider: "test"} | |||
| 19 | + | events <- provider.EventUsage{Usage: provider.Usage{ | |||
| 20 | + | InputTokens: 12, | |||
| 21 | + | OutputTokens: 9, | |||
| 22 | + | ReasoningTokens: 6, | |||
| 23 | + | ReasoningTokensKnown: true, | |||
| 24 | + | }} | |||
| 25 | + | events <- provider.EventDone{ | |||
| 26 | + | Stop: provider.StopEnd, | |||
| 27 | + | Message: provider.Message{Role: provider.RoleAssistant, Content: []provider.Content{ | |||
| 28 | + | provider.TextBlock{Text: "answer"}, | |||
| 29 | + | }}, | |||
| 30 | + | } | |||
| 31 | + | close(events) | |||
| 32 | + | return events, nil | |||
| 33 | + | } | |||
| 34 | + | ||||
| 35 | + | func TestRunPrintReturnsInvocationUsage(t *testing.T) { | |||
| 36 | + | agent := core.NewAgent(printTestClient{}, "test-model", "", core.Registry{}) | |||
| 37 | + | agent.SeedCost(provider.Usage{InputTokens: 100, OutputTokens: 50}) | |||
| 38 | + | var output bytes.Buffer | |||
| 39 | + | ||||
| 40 | + | usage, err := RunPrint(context.Background(), agent, "question", nil, &output) | |||
| 41 | + | if err != nil { | |||
| 42 | + | t.Fatal(err) | |||
| 43 | + | } | |||
| 44 | + | if output.String() != "answer\n" { | |||
| 45 | + | t.Fatalf("output = %q", output.String()) | |||
| 46 | + | } | |||
| 47 | + | if usage.InputTokens != 12 || usage.OutputTokens != 9 || usage.ReasoningTokens != 6 || !usage.ReasoningTokensKnown { | |||
| 48 | + | t.Fatalf("usage = %+v", usage) | |||
| 49 | + | } | |||
| 50 | + | } | |||
A packages/agent/print_stats_test.go +67 -0
| @@ -0,0 +1,67 @@ | |||
| 1 | + | package agent | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "encoding/json" | |
| 5 | + | "os" | |
| 6 | + | "path/filepath" | |
| 7 | + | "testing" | |
| 8 | + | "time" | |
| 9 | + | ||
| 10 | + | "github.com/patriceckhart/zot/packages/provider" | |
| 11 | + | ) | |
| 12 | + | ||
| 13 | + | func TestWritePrintStats(t *testing.T) { | |
| 14 | + | path := filepath.Join(t.TempDir(), "stats.json") | |
| 15 | + | usage := provider.Usage{ | |
| 16 | + | InputTokens: 100, | |
| 17 | + | OutputTokens: 40, | |
| 18 | + | ReasoningTokens: 15, | |
| 19 | + | ReasoningTokensKnown: true, | |
| 20 | + | CacheReadTokens: 20, | |
| 21 | + | CacheWriteTokens: 5, | |
| 22 | + | } | |
| 23 | + | if err := writePrintStats(path, "openai", "gpt-test", usage, 1500*time.Millisecond); err != nil { | |
| 24 | + | t.Fatal(err) | |
| 25 | + | } | |
| 26 | + | ||
| 27 | + | data, err := os.ReadFile(path) | |
| 28 | + | if err != nil { | |
| 29 | + | t.Fatal(err) | |
| 30 | + | } | |
| 31 | + | var got printStats | |
| 32 | + | if err := json.Unmarshal(data, &got); err != nil { | |
| 33 | + | t.Fatal(err) | |
| 34 | + | } | |
| 35 | + | if got.Provider != "openai" || got.Model != "gpt-test" { | |
| 36 | + | t.Fatalf("provider/model = %q/%q", got.Provider, got.Model) | |
| 37 | + | } | |
| 38 | + | if got.PromptTokens != 125 || got.ReasoningTokens == nil || *got.ReasoningTokens != 15 { | |
| 39 | + | t.Fatalf("token stats = %+v", got) | |
| 40 | + | } | |
| 41 | + | if got.GeneratedOutputTokens != 25 || got.ElapsedMS != 1500 { | |
| 42 | + | t.Fatalf("output/time stats = %+v", got) | |
| 43 | + | } | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | func TestWritePrintStatsUsesNullForUnreportedReasoning(t *testing.T) { | |
| 47 | + | path := filepath.Join(t.TempDir(), "stats.json") | |
| 48 | + | usage := provider.Usage{InputTokens: 10, OutputTokens: 7} | |
| 49 | + | if err := writePrintStats(path, "anthropic", "claude-test", usage, time.Second); err != nil { | |
| 50 | + | t.Fatal(err) | |
| 51 | + | } | |
| 52 | + | ||
| 53 | + | data, err := os.ReadFile(path) | |
| 54 | + | if err != nil { | |
| 55 | + | t.Fatal(err) | |
| 56 | + | } | |
| 57 | + | var got map[string]any | |
| 58 | + | if err := json.Unmarshal(data, &got); err != nil { | |
| 59 | + | t.Fatal(err) | |
| 60 | + | } | |
| 61 | + | if got["reasoning_tokens"] != nil { | |
| 62 | + | t.Fatalf("reasoning_tokens = %#v, want null", got["reasoning_tokens"]) | |
| 63 | + | } | |
| 64 | + | if got["generated_output_tokens"] != float64(7) { | |
| 65 | + | t.Fatalf("generated_output_tokens = %#v", got["generated_output_tokens"]) | |
| 66 | + | } | |
| 67 | + | } | |
| @@ -0,0 +1,67 @@ | |||||
| 1 | + | package agent | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "encoding/json" | |||
| 5 | + | "os" | |||
| 6 | + | "path/filepath" | |||
| 7 | + | "testing" | |||
| 8 | + | "time" | |||
| 9 | + | ||||
| 10 | + | "github.com/patriceckhart/zot/packages/provider" | |||
| 11 | + | ) | |||
| 12 | + | ||||
| 13 | + | func TestWritePrintStats(t *testing.T) { | |||
| 14 | + | path := filepath.Join(t.TempDir(), "stats.json") | |||
| 15 | + | usage := provider.Usage{ | |||
| 16 | + | InputTokens: 100, | |||
| 17 | + | OutputTokens: 40, | |||
| 18 | + | ReasoningTokens: 15, | |||
| 19 | + | ReasoningTokensKnown: true, | |||
| 20 | + | CacheReadTokens: 20, | |||
| 21 | + | CacheWriteTokens: 5, | |||
| 22 | + | } | |||
| 23 | + | if err := writePrintStats(path, "openai", "gpt-test", usage, 1500*time.Millisecond); err != nil { | |||
| 24 | + | t.Fatal(err) | |||
| 25 | + | } | |||
| 26 | + | ||||
| 27 | + | data, err := os.ReadFile(path) | |||
| 28 | + | if err != nil { | |||
| 29 | + | t.Fatal(err) | |||
| 30 | + | } | |||
| 31 | + | var got printStats | |||
| 32 | + | if err := json.Unmarshal(data, &got); err != nil { | |||
| 33 | + | t.Fatal(err) | |||
| 34 | + | } | |||
| 35 | + | if got.Provider != "openai" || got.Model != "gpt-test" { | |||
| 36 | + | t.Fatalf("provider/model = %q/%q", got.Provider, got.Model) | |||
| 37 | + | } | |||
| 38 | + | if got.PromptTokens != 125 || got.ReasoningTokens == nil || *got.ReasoningTokens != 15 { | |||
| 39 | + | t.Fatalf("token stats = %+v", got) | |||
| 40 | + | } | |||
| 41 | + | if got.GeneratedOutputTokens != 25 || got.ElapsedMS != 1500 { | |||
| 42 | + | t.Fatalf("output/time stats = %+v", got) | |||
| 43 | + | } | |||
| 44 | + | } | |||
| 45 | + | ||||
| 46 | + | func TestWritePrintStatsUsesNullForUnreportedReasoning(t *testing.T) { | |||
| 47 | + | path := filepath.Join(t.TempDir(), "stats.json") | |||
| 48 | + | usage := provider.Usage{InputTokens: 10, OutputTokens: 7} | |||
| 49 | + | if err := writePrintStats(path, "anthropic", "claude-test", usage, time.Second); err != nil { | |||
| 50 | + | t.Fatal(err) | |||
| 51 | + | } | |||
| 52 | + | ||||
| 53 | + | data, err := os.ReadFile(path) | |||
| 54 | + | if err != nil { | |||
| 55 | + | t.Fatal(err) | |||
| 56 | + | } | |||
| 57 | + | var got map[string]any | |||
| 58 | + | if err := json.Unmarshal(data, &got); err != nil { | |||
| 59 | + | t.Fatal(err) | |||
| 60 | + | } | |||
| 61 | + | if got["reasoning_tokens"] != nil { | |||
| 62 | + | t.Fatalf("reasoning_tokens = %#v, want null", got["reasoning_tokens"]) | |||
| 63 | + | } | |||
| 64 | + | if got["generated_output_tokens"] != float64(7) { | |||
| 65 | + | t.Fatalf("generated_output_tokens = %#v", got["generated_output_tokens"]) | |||
| 66 | + | } | |||
| 67 | + | } | |||
M packages/agent/rpc.go +8 -0
| @@ -414,6 +414,7 @@ func (s *rpcServer) snapshotState() map[string]any { | |||
| 414 | 414 | "usage": map[string]any{ | |
| 415 | 415 | "input": cum.InputTokens, | |
| 416 | 416 | "output": cum.OutputTokens, | |
| 417 | + | "reasoning": rpcReasoningTokens(cum), | |
| 417 | 418 | "cache_read": cum.CacheReadTokens, | |
| 418 | 419 | "cache_write": cum.CacheWriteTokens, | |
| 419 | 420 | "cost_usd": cum.CostUSD, | |
| @@ -421,6 +422,13 @@ func (s *rpcServer) snapshotState() map[string]any { | |||
| 421 | 422 | } | |
| 422 | 423 | } | |
| 423 | 424 | ||
| 425 | + | func rpcReasoningTokens(usage provider.Usage) any { | |
| 426 | + | if !usage.ReasoningTokensKnown { | |
| 427 | + | return nil | |
| 428 | + | } | |
| 429 | + | return usage.ReasoningTokens | |
| 430 | + | } | |
| 431 | + | ||
| 424 | 432 | // ---- write helpers (single-line JSON, mutex-guarded) ---- | |
| 425 | 433 | ||
| 426 | 434 | func (s *rpcServer) writeResponse(id, cmd string, data any) { | |
| @@ -414,6 +414,7 @@ func (s *rpcServer) snapshotState() map[string]any { | |||||
| 414 | "usage": map[string]any{ | 414 | "usage": map[string]any{ | ||
| 415 | "input": cum.InputTokens, | 415 | "input": cum.InputTokens, | ||
| 416 | "output": cum.OutputTokens, | 416 | "output": cum.OutputTokens, | ||
| 417 | + | "reasoning": rpcReasoningTokens(cum), | |||
| 417 | "cache_read": cum.CacheReadTokens, | 418 | "cache_read": cum.CacheReadTokens, | ||
| 418 | "cache_write": cum.CacheWriteTokens, | 419 | "cache_write": cum.CacheWriteTokens, | ||
| 419 | "cost_usd": cum.CostUSD, | 420 | "cost_usd": cum.CostUSD, | ||
| @@ -421,6 +422,13 @@ func (s *rpcServer) snapshotState() map[string]any { | |||||
| 421 | } | 422 | } | ||
| 422 | } | 423 | } | ||
| 423 | 424 | ||||
| 425 | + | func rpcReasoningTokens(usage provider.Usage) any { | |||
| 426 | + | if !usage.ReasoningTokensKnown { | |||
| 427 | + | return nil | |||
| 428 | + | } | |||
| 429 | + | return usage.ReasoningTokens | |||
| 430 | + | } | |||
| 431 | + | ||||
| 424 | // ---- write helpers (single-line JSON, mutex-guarded) ---- | 432 | // ---- write helpers (single-line JSON, mutex-guarded) ---- | ||
| 425 | 433 | ||||
| 426 | func (s *rpcServer) writeResponse(id, cmd string, data any) { | 434 | func (s *rpcServer) writeResponse(id, cmd string, data any) { | ||
M packages/agent/sdk/sdk.go +1 -0
| @@ -191,6 +191,7 @@ func (r *Runtime) Cost() Usage { | |||
| 191 | 191 | return Usage{ | |
| 192 | 192 | Input: u.InputTokens, | |
| 193 | 193 | Output: u.OutputTokens, | |
| 194 | + | Reasoning: sdkReasoningTokens(u), | |
| 194 | 195 | CacheRead: u.CacheReadTokens, | |
| 195 | 196 | CacheWrite: u.CacheWriteTokens, | |
| 196 | 197 | CostUSD: u.CostUSD, | |
| @@ -191,6 +191,7 @@ func (r *Runtime) Cost() Usage { | |||||
| 191 | return Usage{ | 191 | return Usage{ | ||
| 192 | Input: u.InputTokens, | 192 | Input: u.InputTokens, | ||
| 193 | Output: u.OutputTokens, | 193 | Output: u.OutputTokens, | ||
| 194 | + | Reasoning: sdkReasoningTokens(u), | |||
| 194 | CacheRead: u.CacheReadTokens, | 195 | CacheRead: u.CacheReadTokens, | ||
| 195 | CacheWrite: u.CacheWriteTokens, | 196 | CacheWrite: u.CacheWriteTokens, | ||
| 196 | CostUSD: u.CostUSD, | 197 | CostUSD: u.CostUSD, | ||
M packages/agent/sdk/types.go +11 -0
| @@ -85,6 +85,7 @@ type Image struct { | |||
| 85 | 85 | type Usage struct { | |
| 86 | 86 | Input int `json:"input"` | |
| 87 | 87 | Output int `json:"output"` | |
| 88 | + | Reasoning *int `json:"reasoning"` | |
| 88 | 89 | CacheRead int `json:"cache_read"` | |
| 89 | 90 | CacheWrite int `json:"cache_write"` | |
| 90 | 91 | CostUSD float64 `json:"cost_usd"` | |
| @@ -117,6 +118,14 @@ type ModelInfo struct { | |||
| 117 | 118 | ||
| 118 | 119 | // ---- internal converters ---- | |
| 119 | 120 | ||
| 121 | + | func sdkReasoningTokens(usage provider.Usage) *int { | |
| 122 | + | if !usage.ReasoningTokensKnown { | |
| 123 | + | return nil | |
| 124 | + | } | |
| 125 | + | count := usage.ReasoningTokens | |
| 126 | + | return &count | |
| 127 | + | } | |
| 128 | + | ||
| 120 | 129 | func toEvent(ev core.AgentEvent) Event { | |
| 121 | 130 | out := Event{Type: ev.Type()} | |
| 122 | 131 | switch e := ev.(type) { | |
| @@ -143,6 +152,7 @@ func toEvent(ev core.AgentEvent) Event { | |||
| 143 | 152 | out.Usage = &Usage{ | |
| 144 | 153 | Input: e.Usage.InputTokens, | |
| 145 | 154 | Output: e.Usage.OutputTokens, | |
| 155 | + | Reasoning: sdkReasoningTokens(e.Usage), | |
| 146 | 156 | CacheRead: e.Usage.CacheReadTokens, | |
| 147 | 157 | CacheWrite: e.Usage.CacheWriteTokens, | |
| 148 | 158 | CostUSD: e.Usage.CostUSD, | |
| @@ -150,6 +160,7 @@ func toEvent(ev core.AgentEvent) Event { | |||
| 150 | 160 | out.Cumulative = &Usage{ | |
| 151 | 161 | Input: e.Cumulative.InputTokens, | |
| 152 | 162 | Output: e.Cumulative.OutputTokens, | |
| 163 | + | Reasoning: sdkReasoningTokens(e.Cumulative), | |
| 153 | 164 | CacheRead: e.Cumulative.CacheReadTokens, | |
| 154 | 165 | CacheWrite: e.Cumulative.CacheWriteTokens, | |
| 155 | 166 | CostUSD: e.Cumulative.CostUSD, | |
| @@ -85,6 +85,7 @@ type Image struct { | |||||
| 85 | type Usage struct { | 85 | type Usage struct { | ||
| 86 | Input int `json:"input"` | 86 | Input int `json:"input"` | ||
| 87 | Output int `json:"output"` | 87 | Output int `json:"output"` | ||
| 88 | + | Reasoning *int `json:"reasoning"` | |||
| 88 | CacheRead int `json:"cache_read"` | 89 | CacheRead int `json:"cache_read"` | ||
| 89 | CacheWrite int `json:"cache_write"` | 90 | CacheWrite int `json:"cache_write"` | ||
| 90 | CostUSD float64 `json:"cost_usd"` | 91 | CostUSD float64 `json:"cost_usd"` | ||
| @@ -117,6 +118,14 @@ type ModelInfo struct { | |||||
| 117 | 118 | ||||
| 118 | // ---- internal converters ---- | 119 | // ---- internal converters ---- | ||
| 119 | 120 | ||||
| 121 | + | func sdkReasoningTokens(usage provider.Usage) *int { | |||
| 122 | + | if !usage.ReasoningTokensKnown { | |||
| 123 | + | return nil | |||
| 124 | + | } | |||
| 125 | + | count := usage.ReasoningTokens | |||
| 126 | + | return &count | |||
| 127 | + | } | |||
| 128 | + | ||||
| 120 | func toEvent(ev core.AgentEvent) Event { | 129 | func toEvent(ev core.AgentEvent) Event { | ||
| 121 | out := Event{Type: ev.Type()} | 130 | out := Event{Type: ev.Type()} | ||
| 122 | switch e := ev.(type) { | 131 | switch e := ev.(type) { | ||
| @@ -143,6 +152,7 @@ func toEvent(ev core.AgentEvent) Event { | |||||
| 143 | out.Usage = &Usage{ | 152 | out.Usage = &Usage{ | ||
| 144 | Input: e.Usage.InputTokens, | 153 | Input: e.Usage.InputTokens, | ||
| 145 | Output: e.Usage.OutputTokens, | 154 | Output: e.Usage.OutputTokens, | ||
| 155 | + | Reasoning: sdkReasoningTokens(e.Usage), | |||
| 146 | CacheRead: e.Usage.CacheReadTokens, | 156 | CacheRead: e.Usage.CacheReadTokens, | ||
| 147 | CacheWrite: e.Usage.CacheWriteTokens, | 157 | CacheWrite: e.Usage.CacheWriteTokens, | ||
| 148 | CostUSD: e.Usage.CostUSD, | 158 | CostUSD: e.Usage.CostUSD, | ||
| @@ -150,6 +160,7 @@ func toEvent(ev core.AgentEvent) Event { | |||||
| 150 | out.Cumulative = &Usage{ | 160 | out.Cumulative = &Usage{ | ||
| 151 | Input: e.Cumulative.InputTokens, | 161 | Input: e.Cumulative.InputTokens, | ||
| 152 | Output: e.Cumulative.OutputTokens, | 162 | Output: e.Cumulative.OutputTokens, | ||
| 163 | + | Reasoning: sdkReasoningTokens(e.Cumulative), | |||
| 153 | CacheRead: e.Cumulative.CacheReadTokens, | 164 | CacheRead: e.Cumulative.CacheReadTokens, | ||
| 154 | CacheWrite: e.Cumulative.CacheWriteTokens, | 165 | CacheWrite: e.Cumulative.CacheWriteTokens, | ||
| 155 | CostUSD: e.Cumulative.CostUSD, | 166 | CostUSD: e.Cumulative.CostUSD, | ||
M packages/core/agent.go +1 -1
| @@ -268,7 +268,7 @@ func (a *Agent) Cost() provider.Usage { | |||
| 268 | 268 | func (a *Agent) SeedCost(u provider.Usage) { | |
| 269 | 269 | a.mu.Lock() | |
| 270 | 270 | defer a.mu.Unlock() | |
| 271 | - | a.cost.Total = u | |
| 271 | + | a.cost.Seed(u) | |
| 272 | 272 | } | |
| 273 | 273 | ||
| 274 | 274 | // LastTurnUsage returns the per-turn usage of the most recent | |
| @@ -268,7 +268,7 @@ func (a *Agent) Cost() provider.Usage { | |||||
| 268 | func (a *Agent) SeedCost(u provider.Usage) { | 268 | func (a *Agent) SeedCost(u provider.Usage) { | ||
| 269 | a.mu.Lock() | 269 | a.mu.Lock() | ||
| 270 | defer a.mu.Unlock() | 270 | defer a.mu.Unlock() | ||
| 271 | - | a.cost.Total = u | 271 | + | a.cost.Seed(u) |
| 272 | } | 272 | } | ||
| 273 | 273 | ||||
| 274 | // LastTurnUsage returns the per-turn usage of the most recent | 274 | // LastTurnUsage returns the per-turn usage of the most recent | ||
M packages/core/core_test.go +5 -2
| @@ -154,11 +154,14 @@ func TestNewSessionAtPathCreatesAtExplicitPath(t *testing.T) { | |||
| 154 | 154 | ||
| 155 | 155 | func TestCostAdd(t *testing.T) { | |
| 156 | 156 | var c CostTracker | |
| 157 | - | c.Add(provider.Usage{InputTokens: 100, OutputTokens: 50, CostUSD: 0.01}) | |
| 158 | - | c.Add(provider.Usage{InputTokens: 200, OutputTokens: 25, CostUSD: 0.02}) | |
| 157 | + | c.Add(provider.Usage{InputTokens: 100, OutputTokens: 50, ReasoningTokens: 20, ReasoningTokensKnown: true, CostUSD: 0.01}) | |
| 158 | + | c.Add(provider.Usage{InputTokens: 200, OutputTokens: 25, ReasoningTokens: 10, ReasoningTokensKnown: true, CostUSD: 0.02}) | |
| 159 | 159 | if c.Total.InputTokens != 300 || c.Total.OutputTokens != 75 { | |
| 160 | 160 | t.Fatalf("got %+v", c.Total) | |
| 161 | 161 | } | |
| 162 | + | if c.Total.ReasoningTokens != 30 || !c.Total.ReasoningTokensKnown { | |
| 163 | + | t.Fatalf("got reasoning usage %+v", c.Total) | |
| 164 | + | } | |
| 162 | 165 | if c.Total.CostUSD < 0.0299 || c.Total.CostUSD > 0.0301 { | |
| 163 | 166 | t.Fatalf("got cost %v", c.Total.CostUSD) | |
| 164 | 167 | } | |
| @@ -154,11 +154,14 @@ func TestNewSessionAtPathCreatesAtExplicitPath(t *testing.T) { | |||||
| 154 | 154 | ||||
| 155 | func TestCostAdd(t *testing.T) { | 155 | func TestCostAdd(t *testing.T) { | ||
| 156 | var c CostTracker | 156 | var c CostTracker | ||
| 157 | - | c.Add(provider.Usage{InputTokens: 100, OutputTokens: 50, CostUSD: 0.01}) | 157 | + | c.Add(provider.Usage{InputTokens: 100, OutputTokens: 50, ReasoningTokens: 20, ReasoningTokensKnown: true, CostUSD: 0.01}) |
| 158 | - | c.Add(provider.Usage{InputTokens: 200, OutputTokens: 25, CostUSD: 0.02}) | 158 | + | c.Add(provider.Usage{InputTokens: 200, OutputTokens: 25, ReasoningTokens: 10, ReasoningTokensKnown: true, CostUSD: 0.02}) |
| 159 | if c.Total.InputTokens != 300 || c.Total.OutputTokens != 75 { | 159 | if c.Total.InputTokens != 300 || c.Total.OutputTokens != 75 { | ||
| 160 | t.Fatalf("got %+v", c.Total) | 160 | t.Fatalf("got %+v", c.Total) | ||
| 161 | } | 161 | } | ||
| 162 | + | if c.Total.ReasoningTokens != 30 || !c.Total.ReasoningTokensKnown { | |||
| 163 | + | t.Fatalf("got reasoning usage %+v", c.Total) | |||
| 164 | + | } | |||
| 162 | if c.Total.CostUSD < 0.0299 || c.Total.CostUSD > 0.0301 { | 165 | if c.Total.CostUSD < 0.0299 || c.Total.CostUSD > 0.0301 { | ||
| 163 | t.Fatalf("got cost %v", c.Total.CostUSD) | 166 | t.Fatalf("got cost %v", c.Total.CostUSD) | ||
| 164 | } | 167 | } | ||
M packages/core/cost.go +13 -1
| @@ -12,12 +12,24 @@ import "github.com/patriceckhart/zot/packages/provider" | |||
| 12 | 12 | type CostTracker struct { | |
| 13 | 13 | Total provider.Usage | |
| 14 | 14 | LastTurn provider.Usage | |
| 15 | + | hasTotal bool | |
| 15 | 16 | } | |
| 16 | 17 | ||
| 17 | 18 | // Add folds u into the running total, records u as the last-turn | |
| 18 | 19 | // snapshot, and returns the new cumulative value. | |
| 19 | 20 | func (c *CostTracker) Add(u provider.Usage) provider.Usage { | |
| 20 | - | c.Total = c.Total.Add(u) | |
| 21 | + | if c.hasTotal { | |
| 22 | + | c.Total = c.Total.Add(u) | |
| 23 | + | } else { | |
| 24 | + | c.Total = u | |
| 25 | + | c.hasTotal = true | |
| 26 | + | } | |
| 21 | 27 | c.LastTurn = u | |
| 22 | 28 | return c.Total | |
| 23 | 29 | } | |
| 30 | + | ||
| 31 | + | // Seed replaces the running total with usage restored from a session. | |
| 32 | + | func (c *CostTracker) Seed(u provider.Usage) { | |
| 33 | + | c.Total = u | |
| 34 | + | c.hasTotal = u.InputTokens != 0 || u.OutputTokens != 0 || u.CacheReadTokens != 0 || u.CacheWriteTokens != 0 || u.CostUSD != 0 | |
| 35 | + | } | |
| @@ -12,12 +12,24 @@ import "github.com/patriceckhart/zot/packages/provider" | |||||
| 12 | type CostTracker struct { | 12 | type CostTracker struct { | ||
| 13 | Total provider.Usage | 13 | Total provider.Usage | ||
| 14 | LastTurn provider.Usage | 14 | LastTurn provider.Usage | ||
| 15 | + | hasTotal bool | |||
| 15 | } | 16 | } | ||
| 16 | 17 | ||||
| 17 | // Add folds u into the running total, records u as the last-turn | 18 | // Add folds u into the running total, records u as the last-turn | ||
| 18 | // snapshot, and returns the new cumulative value. | 19 | // snapshot, and returns the new cumulative value. | ||
| 19 | func (c *CostTracker) Add(u provider.Usage) provider.Usage { | 20 | func (c *CostTracker) Add(u provider.Usage) provider.Usage { | ||
| 20 | - | c.Total = c.Total.Add(u) | 21 | + | if c.hasTotal { |
| 22 | + | c.Total = c.Total.Add(u) | |||
| 23 | + | } else { | |||
| 24 | + | c.Total = u | |||
| 25 | + | c.hasTotal = true | |||
| 26 | + | } | |||
| 21 | c.LastTurn = u | 27 | c.LastTurn = u | ||
| 22 | return c.Total | 28 | return c.Total | ||
| 23 | } | 29 | } | ||
| 30 | + | ||||
| 31 | + | // Seed replaces the running total with usage restored from a session. | |||
| 32 | + | func (c *CostTracker) Seed(u provider.Usage) { | |||
| 33 | + | c.Total = u | |||
| 34 | + | c.hasTotal = u.InputTokens != 0 || u.OutputTokens != 0 || u.CacheReadTokens != 0 || u.CacheWriteTokens != 0 || u.CostUSD != 0 | |||
| 35 | + | } | |||
M packages/core/session.go +2 -0
| @@ -213,6 +213,8 @@ func SessionUsageDetail(path string) (cumulative, lastTurn provider.Usage, err e | |||
| 213 | 213 | // input/output are monotonic totals -> per-turn = delta. | |
| 214 | 214 | lastTurn.InputTokens = nonNegDelta(cumulative.InputTokens, prevCum.InputTokens) | |
| 215 | 215 | lastTurn.OutputTokens = nonNegDelta(cumulative.OutputTokens, prevCum.OutputTokens) | |
| 216 | + | lastTurn.ReasoningTokens = nonNegDelta(cumulative.ReasoningTokens, prevCum.ReasoningTokens) | |
| 217 | + | lastTurn.ReasoningTokensKnown = cumulative.ReasoningTokensKnown | |
| 216 | 218 | // cache_read/write on the final row already represent the last prompt's | |
| 217 | 219 | // cache hit/creation, not a running total of bytes; use directly. | |
| 218 | 220 | lastTurn.CacheReadTokens = cumulative.CacheReadTokens - prevCum.CacheReadTokens | |
| @@ -213,6 +213,8 @@ func SessionUsageDetail(path string) (cumulative, lastTurn provider.Usage, err e | |||||
| 213 | // input/output are monotonic totals -> per-turn = delta. | 213 | // input/output are monotonic totals -> per-turn = delta. | ||
| 214 | lastTurn.InputTokens = nonNegDelta(cumulative.InputTokens, prevCum.InputTokens) | 214 | lastTurn.InputTokens = nonNegDelta(cumulative.InputTokens, prevCum.InputTokens) | ||
| 215 | lastTurn.OutputTokens = nonNegDelta(cumulative.OutputTokens, prevCum.OutputTokens) | 215 | lastTurn.OutputTokens = nonNegDelta(cumulative.OutputTokens, prevCum.OutputTokens) | ||
| 216 | + | lastTurn.ReasoningTokens = nonNegDelta(cumulative.ReasoningTokens, prevCum.ReasoningTokens) | |||
| 217 | + | lastTurn.ReasoningTokensKnown = cumulative.ReasoningTokensKnown | |||
| 216 | // cache_read/write on the final row already represent the last prompt's | 218 | // cache_read/write on the final row already represent the last prompt's | ||
| 217 | // cache hit/creation, not a running total of bytes; use directly. | 219 | // cache hit/creation, not a running total of bytes; use directly. | ||
| 218 | lastTurn.CacheReadTokens = cumulative.CacheReadTokens - prevCum.CacheReadTokens | 220 | lastTurn.CacheReadTokens = cumulative.CacheReadTokens - prevCum.CacheReadTokens | ||
M packages/provider/gemini.go +11 -6
| @@ -677,11 +677,11 @@ func (c *geminiClient) runStream(ctx context.Context, resp *http.Response, req R | |||
| 677 | 677 | FinishReason string `json:"finishReason"` | |
| 678 | 678 | } `json:"candidates"` | |
| 679 | 679 | UsageMetadata *struct { | |
| 680 | - | PromptTokenCount int `json:"promptTokenCount"` | |
| 681 | - | CandidatesTokenCount int `json:"candidatesTokenCount"` | |
| 682 | - | ThoughtsTokenCount int `json:"thoughtsTokenCount"` | |
| 683 | - | CachedContentTokenCount int `json:"cachedContentTokenCount"` | |
| 684 | - | TotalTokenCount int `json:"totalTokenCount"` | |
| 680 | + | PromptTokenCount int `json:"promptTokenCount"` | |
| 681 | + | CandidatesTokenCount int `json:"candidatesTokenCount"` | |
| 682 | + | ThoughtsTokenCount *int `json:"thoughtsTokenCount"` | |
| 683 | + | CachedContentTokenCount int `json:"cachedContentTokenCount"` | |
| 684 | + | TotalTokenCount int `json:"totalTokenCount"` | |
| 685 | 685 | } `json:"usageMetadata"` | |
| 686 | 686 | Error *struct { | |
| 687 | 687 | Code int `json:"code"` | |
| @@ -753,7 +753,12 @@ func (c *geminiClient) runStream(ctx context.Context, resp *http.Response, req R | |||
| 753 | 753 | input = um.PromptTokenCount | |
| 754 | 754 | } | |
| 755 | 755 | usage.InputTokens = input | |
| 756 | - | usage.OutputTokens = um.CandidatesTokenCount + um.ThoughtsTokenCount | |
| 756 | + | usage.OutputTokens = um.CandidatesTokenCount | |
| 757 | + | if um.ThoughtsTokenCount != nil { | |
| 758 | + | usage.ReasoningTokens = *um.ThoughtsTokenCount | |
| 759 | + | usage.ReasoningTokensKnown = true | |
| 760 | + | usage.OutputTokens += *um.ThoughtsTokenCount | |
| 761 | + | } | |
| 757 | 762 | usage.CacheReadTokens = um.CachedContentTokenCount | |
| 758 | 763 | } | |
| 759 | 764 | // Promote ToolUse stop when tool calls are present and | |
| @@ -677,11 +677,11 @@ func (c *geminiClient) runStream(ctx context.Context, resp *http.Response, req R | |||||
| 677 | FinishReason string `json:"finishReason"` | 677 | FinishReason string `json:"finishReason"` | ||
| 678 | } `json:"candidates"` | 678 | } `json:"candidates"` | ||
| 679 | UsageMetadata *struct { | 679 | UsageMetadata *struct { | ||
| 680 | - | PromptTokenCount int `json:"promptTokenCount"` | 680 | + | PromptTokenCount int `json:"promptTokenCount"` |
| 681 | - | CandidatesTokenCount int `json:"candidatesTokenCount"` | 681 | + | CandidatesTokenCount int `json:"candidatesTokenCount"` |
| 682 | - | ThoughtsTokenCount int `json:"thoughtsTokenCount"` | 682 | + | ThoughtsTokenCount *int `json:"thoughtsTokenCount"` |
| 683 | - | CachedContentTokenCount int `json:"cachedContentTokenCount"` | 683 | + | CachedContentTokenCount int `json:"cachedContentTokenCount"` |
| 684 | - | TotalTokenCount int `json:"totalTokenCount"` | 684 | + | TotalTokenCount int `json:"totalTokenCount"` |
| 685 | } `json:"usageMetadata"` | 685 | } `json:"usageMetadata"` | ||
| 686 | Error *struct { | 686 | Error *struct { | ||
| 687 | Code int `json:"code"` | 687 | Code int `json:"code"` | ||
| @@ -753,7 +753,12 @@ func (c *geminiClient) runStream(ctx context.Context, resp *http.Response, req R | |||||
| 753 | input = um.PromptTokenCount | 753 | input = um.PromptTokenCount | ||
| 754 | } | 754 | } | ||
| 755 | usage.InputTokens = input | 755 | usage.InputTokens = input | ||
| 756 | - | usage.OutputTokens = um.CandidatesTokenCount + um.ThoughtsTokenCount | 756 | + | usage.OutputTokens = um.CandidatesTokenCount |
| 757 | + | if um.ThoughtsTokenCount != nil { | |||
| 758 | + | usage.ReasoningTokens = *um.ThoughtsTokenCount | |||
| 759 | + | usage.ReasoningTokensKnown = true | |||
| 760 | + | usage.OutputTokens += *um.ThoughtsTokenCount | |||
| 761 | + | } | |||
| 757 | usage.CacheReadTokens = um.CachedContentTokenCount | 762 | usage.CacheReadTokens = um.CachedContentTokenCount | ||
| 758 | } | 763 | } | ||
| 759 | // Promote ToolUse stop when tool calls are present and | 764 | // Promote ToolUse stop when tool calls are present and | ||
M packages/provider/gemini_test.go +3 -0
| @@ -510,6 +510,9 @@ func TestGeminiStreamReasoning(t *testing.T) { | |||
| 510 | 510 | if usage.OutputTokens != 5 { // candidatesTokenCount (2) + thoughtsTokenCount (3) | |
| 511 | 511 | t.Fatalf("usage output tokens=%d, want 5", usage.OutputTokens) | |
| 512 | 512 | } | |
| 513 | + | if usage.ReasoningTokens != 3 || !usage.ReasoningTokensKnown { | |
| 514 | + | t.Fatalf("usage reasoning tokens=%d known=%v, want 3 true", usage.ReasoningTokens, usage.ReasoningTokensKnown) | |
| 515 | + | } | |
| 513 | 516 | ||
| 514 | 517 | // Verify that the final message contains both the ReasoningBlock and the TextBlock | |
| 515 | 518 | msg := done.Message | |
| @@ -510,6 +510,9 @@ func TestGeminiStreamReasoning(t *testing.T) { | |||||
| 510 | if usage.OutputTokens != 5 { // candidatesTokenCount (2) + thoughtsTokenCount (3) | 510 | if usage.OutputTokens != 5 { // candidatesTokenCount (2) + thoughtsTokenCount (3) | ||
| 511 | t.Fatalf("usage output tokens=%d, want 5", usage.OutputTokens) | 511 | t.Fatalf("usage output tokens=%d, want 5", usage.OutputTokens) | ||
| 512 | } | 512 | } | ||
| 513 | + | if usage.ReasoningTokens != 3 || !usage.ReasoningTokensKnown { | |||
| 514 | + | t.Fatalf("usage reasoning tokens=%d known=%v, want 3 true", usage.ReasoningTokens, usage.ReasoningTokensKnown) | |||
| 515 | + | } | |||
| 513 | 516 | ||||
| 514 | // Verify that the final message contains both the ReasoningBlock and the TextBlock | 517 | // Verify that the final message contains both the ReasoningBlock and the TextBlock | ||
| 515 | msg := done.Message | 518 | msg := done.Message | ||
M packages/provider/openai.go +7 -0
| @@ -633,6 +633,9 @@ func (c *openaiClient) runStream(ctx context.Context, resp *http.Response, req R | |||
| 633 | 633 | PromptTokensDetails struct { | |
| 634 | 634 | CachedTokens int `json:"cached_tokens"` | |
| 635 | 635 | } `json:"prompt_tokens_details"` | |
| 636 | + | CompletionTokensDetails *struct { | |
| 637 | + | ReasoningTokens int `json:"reasoning_tokens"` | |
| 638 | + | } `json:"completion_tokens_details"` | |
| 636 | 639 | } `json:"usage"` | |
| 637 | 640 | Error *struct { | |
| 638 | 641 | Message string `json:"message"` | |
| @@ -655,6 +658,10 @@ func (c *openaiClient) runStream(ctx context.Context, resp *http.Response, req R | |||
| 655 | 658 | } | |
| 656 | 659 | usage.OutputTokens = chunk.Usage.CompletionTokens | |
| 657 | 660 | usage.CacheReadTokens = chunk.Usage.PromptTokensDetails.CachedTokens | |
| 661 | + | if details := chunk.Usage.CompletionTokensDetails; details != nil { | |
| 662 | + | usage.ReasoningTokens = details.ReasoningTokens | |
| 663 | + | usage.ReasoningTokensKnown = true | |
| 664 | + | } | |
| 658 | 665 | } | |
| 659 | 666 | for _, ch := range chunk.Choices { | |
| 660 | 667 | if ch.Delta.ReasoningContent != "" { | |
| @@ -633,6 +633,9 @@ func (c *openaiClient) runStream(ctx context.Context, resp *http.Response, req R | |||||
| 633 | PromptTokensDetails struct { | 633 | PromptTokensDetails struct { | ||
| 634 | CachedTokens int `json:"cached_tokens"` | 634 | CachedTokens int `json:"cached_tokens"` | ||
| 635 | } `json:"prompt_tokens_details"` | 635 | } `json:"prompt_tokens_details"` | ||
| 636 | + | CompletionTokensDetails *struct { | |||
| 637 | + | ReasoningTokens int `json:"reasoning_tokens"` | |||
| 638 | + | } `json:"completion_tokens_details"` | |||
| 636 | } `json:"usage"` | 639 | } `json:"usage"` | ||
| 637 | Error *struct { | 640 | Error *struct { | ||
| 638 | Message string `json:"message"` | 641 | Message string `json:"message"` | ||
| @@ -655,6 +658,10 @@ func (c *openaiClient) runStream(ctx context.Context, resp *http.Response, req R | |||||
| 655 | } | 658 | } | ||
| 656 | usage.OutputTokens = chunk.Usage.CompletionTokens | 659 | usage.OutputTokens = chunk.Usage.CompletionTokens | ||
| 657 | usage.CacheReadTokens = chunk.Usage.PromptTokensDetails.CachedTokens | 660 | usage.CacheReadTokens = chunk.Usage.PromptTokensDetails.CachedTokens | ||
| 661 | + | if details := chunk.Usage.CompletionTokensDetails; details != nil { | |||
| 662 | + | usage.ReasoningTokens = details.ReasoningTokens | |||
| 663 | + | usage.ReasoningTokensKnown = true | |||
| 664 | + | } | |||
| 658 | } | 665 | } | ||
| 659 | for _, ch := range chunk.Choices { | 666 | for _, ch := range chunk.Choices { | ||
| 660 | if ch.Delta.ReasoningContent != "" { | 667 | if ch.Delta.ReasoningContent != "" { | ||
M packages/provider/openai_codex.go +7 -0
| @@ -622,6 +622,9 @@ func (c *codexClient) runStream(ctx context.Context, resp *http.Response, req Re | |||
| 622 | 622 | InputTokensDetails struct { | |
| 623 | 623 | CachedTokens int `json:"cached_tokens"` | |
| 624 | 624 | } `json:"input_tokens_details"` | |
| 625 | + | OutputTokensDetails *struct { | |
| 626 | + | ReasoningTokens int `json:"reasoning_tokens"` | |
| 627 | + | } `json:"output_tokens_details"` | |
| 625 | 628 | } `json:"usage"` | |
| 626 | 629 | Status string `json:"status"` | |
| 627 | 630 | } `json:"response"` | |
| @@ -633,6 +636,10 @@ func (c *codexClient) runStream(ctx context.Context, resp *http.Response, req Re | |||
| 633 | 636 | } | |
| 634 | 637 | usage.OutputTokens = p.Response.Usage.OutputTokens | |
| 635 | 638 | usage.CacheReadTokens = p.Response.Usage.InputTokensDetails.CachedTokens | |
| 639 | + | if details := p.Response.Usage.OutputTokensDetails; details != nil { | |
| 640 | + | usage.ReasoningTokens = details.ReasoningTokens | |
| 641 | + | usage.ReasoningTokensKnown = true | |
| 642 | + | } | |
| 636 | 643 | ||
| 637 | 644 | hadTool := false | |
| 638 | 645 | for _, it := range items { | |
| @@ -622,6 +622,9 @@ func (c *codexClient) runStream(ctx context.Context, resp *http.Response, req Re | |||||
| 622 | InputTokensDetails struct { | 622 | InputTokensDetails struct { | ||
| 623 | CachedTokens int `json:"cached_tokens"` | 623 | CachedTokens int `json:"cached_tokens"` | ||
| 624 | } `json:"input_tokens_details"` | 624 | } `json:"input_tokens_details"` | ||
| 625 | + | OutputTokensDetails *struct { | |||
| 626 | + | ReasoningTokens int `json:"reasoning_tokens"` | |||
| 627 | + | } `json:"output_tokens_details"` | |||
| 625 | } `json:"usage"` | 628 | } `json:"usage"` | ||
| 626 | Status string `json:"status"` | 629 | Status string `json:"status"` | ||
| 627 | } `json:"response"` | 630 | } `json:"response"` | ||
| @@ -633,6 +636,10 @@ func (c *codexClient) runStream(ctx context.Context, resp *http.Response, req Re | |||||
| 633 | } | 636 | } | ||
| 634 | usage.OutputTokens = p.Response.Usage.OutputTokens | 637 | usage.OutputTokens = p.Response.Usage.OutputTokens | ||
| 635 | usage.CacheReadTokens = p.Response.Usage.InputTokensDetails.CachedTokens | 638 | usage.CacheReadTokens = p.Response.Usage.InputTokensDetails.CachedTokens | ||
| 639 | + | if details := p.Response.Usage.OutputTokensDetails; details != nil { | |||
| 640 | + | usage.ReasoningTokens = details.ReasoningTokens | |||
| 641 | + | usage.ReasoningTokensKnown = true | |||
| 642 | + | } | |||
| 636 | 643 | ||||
| 637 | hadTool := false | 644 | hadTool := false | ||
| 638 | for _, it := range items { | 645 | for _, it := range items { | ||
M packages/provider/provider.go +16 -11
| @@ -156,21 +156,26 @@ func activatedToolNames(messages []Message) map[string]bool { | |||
| 156 | 156 | ||
| 157 | 157 | // Usage aggregates token counts and cost for a turn. | |
| 158 | 158 | type Usage struct { | |
| 159 | - | InputTokens int `json:"input_tokens"` | |
| 160 | - | OutputTokens int `json:"output_tokens"` | |
| 161 | - | CacheReadTokens int `json:"cache_read_tokens"` | |
| 162 | - | CacheWriteTokens int `json:"cache_write_tokens"` | |
| 163 | - | CostUSD float64 `json:"cost_usd"` | |
| 159 | + | InputTokens int `json:"input_tokens"` | |
| 160 | + | OutputTokens int `json:"output_tokens"` | |
| 161 | + | ReasoningTokens int `json:"reasoning_tokens"` | |
| 162 | + | ReasoningTokensKnown bool `json:"reasoning_tokens_known,omitempty"` | |
| 163 | + | CacheReadTokens int `json:"cache_read_tokens"` | |
| 164 | + | CacheWriteTokens int `json:"cache_write_tokens"` | |
| 165 | + | CostUSD float64 `json:"cost_usd"` | |
| 164 | 166 | } | |
| 165 | 167 | ||
| 166 | - | // Add returns u plus v. | |
| 168 | + | // Add returns u plus v. A reasoning total is known only when both | |
| 169 | + | // component usage reports include a separate reasoning-token count. | |
| 167 | 170 | func (u Usage) Add(v Usage) Usage { | |
| 168 | 171 | return Usage{ | |
| 169 | - | InputTokens: u.InputTokens + v.InputTokens, | |
| 170 | - | OutputTokens: u.OutputTokens + v.OutputTokens, | |
| 171 | - | CacheReadTokens: u.CacheReadTokens + v.CacheReadTokens, | |
| 172 | - | CacheWriteTokens: u.CacheWriteTokens + v.CacheWriteTokens, | |
| 173 | - | CostUSD: u.CostUSD + v.CostUSD, | |
| 172 | + | InputTokens: u.InputTokens + v.InputTokens, | |
| 173 | + | OutputTokens: u.OutputTokens + v.OutputTokens, | |
| 174 | + | ReasoningTokens: u.ReasoningTokens + v.ReasoningTokens, | |
| 175 | + | ReasoningTokensKnown: u.ReasoningTokensKnown && v.ReasoningTokensKnown, | |
| 176 | + | CacheReadTokens: u.CacheReadTokens + v.CacheReadTokens, | |
| 177 | + | CacheWriteTokens: u.CacheWriteTokens + v.CacheWriteTokens, | |
| 178 | + | CostUSD: u.CostUSD + v.CostUSD, | |
| 174 | 179 | } | |
| 175 | 180 | } | |
| 176 | 181 | ||
| @@ -156,21 +156,26 @@ func activatedToolNames(messages []Message) map[string]bool { | |||||
| 156 | 156 | ||||
| 157 | // Usage aggregates token counts and cost for a turn. | 157 | // Usage aggregates token counts and cost for a turn. | ||
| 158 | type Usage struct { | 158 | type Usage struct { | ||
| 159 | - | InputTokens int `json:"input_tokens"` | 159 | + | InputTokens int `json:"input_tokens"` |
| 160 | - | OutputTokens int `json:"output_tokens"` | 160 | + | OutputTokens int `json:"output_tokens"` |
| 161 | - | CacheReadTokens int `json:"cache_read_tokens"` | 161 | + | ReasoningTokens int `json:"reasoning_tokens"` |
| 162 | - | CacheWriteTokens int `json:"cache_write_tokens"` | 162 | + | ReasoningTokensKnown bool `json:"reasoning_tokens_known,omitempty"` |
| 163 | - | CostUSD float64 `json:"cost_usd"` | 163 | + | CacheReadTokens int `json:"cache_read_tokens"` |
| 164 | + | CacheWriteTokens int `json:"cache_write_tokens"` | |||
| 165 | + | CostUSD float64 `json:"cost_usd"` | |||
| 164 | } | 166 | } | ||
| 165 | 167 | ||||
| 166 | - | // Add returns u plus v. | 168 | + | // Add returns u plus v. A reasoning total is known only when both |
| 169 | + | // component usage reports include a separate reasoning-token count. | |||
| 167 | func (u Usage) Add(v Usage) Usage { | 170 | func (u Usage) Add(v Usage) Usage { | ||
| 168 | return Usage{ | 171 | return Usage{ | ||
| 169 | - | InputTokens: u.InputTokens + v.InputTokens, | 172 | + | InputTokens: u.InputTokens + v.InputTokens, |
| 170 | - | OutputTokens: u.OutputTokens + v.OutputTokens, | 173 | + | OutputTokens: u.OutputTokens + v.OutputTokens, |
| 171 | - | CacheReadTokens: u.CacheReadTokens + v.CacheReadTokens, | 174 | + | ReasoningTokens: u.ReasoningTokens + v.ReasoningTokens, |
| 172 | - | CacheWriteTokens: u.CacheWriteTokens + v.CacheWriteTokens, | 175 | + | ReasoningTokensKnown: u.ReasoningTokensKnown && v.ReasoningTokensKnown, |
| 173 | - | CostUSD: u.CostUSD + v.CostUSD, | 176 | + | CacheReadTokens: u.CacheReadTokens + v.CacheReadTokens, |
| 177 | + | CacheWriteTokens: u.CacheWriteTokens + v.CacheWriteTokens, | |||
| 178 | + | CostUSD: u.CostUSD + v.CostUSD, | |||
| 174 | } | 179 | } | ||
| 175 | } | 180 | } | ||
| 176 | 181 | ||||
A packages/provider/reasoning_usage_test.go +57 -0
| @@ -0,0 +1,57 @@ | |||
| 1 | + | package provider | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "fmt" | |
| 6 | + | "net/http" | |
| 7 | + | "net/http/httptest" | |
| 8 | + | "testing" | |
| 9 | + | ) | |
| 10 | + | ||
| 11 | + | func TestOpenAIReportsReasoningTokens(t *testing.T) { | |
| 12 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 13 | + | w.Header().Set("content-type", "text/event-stream") | |
| 14 | + | fmt.Fprintln(w, `data: {"choices":[{"delta":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":12,"completion_tokens":9,"completion_tokens_details":{"reasoning_tokens":6}}}`) | |
| 15 | + | fmt.Fprintln(w) | |
| 16 | + | fmt.Fprintln(w, "data: [DONE]") | |
| 17 | + | fmt.Fprintln(w) | |
| 18 | + | })) | |
| 19 | + | defer srv.Close() | |
| 20 | + | ||
| 21 | + | events, err := NewOpenAI("test", srv.URL).Stream(context.Background(), Request{Model: "gpt-test"}) | |
| 22 | + | if err != nil { | |
| 23 | + | t.Fatal(err) | |
| 24 | + | } | |
| 25 | + | usage := usageFromEvents(events) | |
| 26 | + | if usage.OutputTokens != 9 || usage.ReasoningTokens != 6 || !usage.ReasoningTokensKnown { | |
| 27 | + | t.Fatalf("usage = %+v", usage) | |
| 28 | + | } | |
| 29 | + | } | |
| 30 | + | ||
| 31 | + | func TestOpenAIResponsesReportsReasoningTokens(t *testing.T) { | |
| 32 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 33 | + | w.Header().Set("content-type", "text/event-stream") | |
| 34 | + | fmt.Fprintln(w, `data: {"type":"response.completed","response":{"status":"completed","usage":{"input_tokens":12,"output_tokens":9,"output_tokens_details":{"reasoning_tokens":6}}}}`) | |
| 35 | + | fmt.Fprintln(w) | |
| 36 | + | })) | |
| 37 | + | defer srv.Close() | |
| 38 | + | ||
| 39 | + | events, err := NewOpenAICodex("test", "account", srv.URL).Stream(context.Background(), Request{Model: "gpt-5.5"}) | |
| 40 | + | if err != nil { | |
| 41 | + | t.Fatal(err) | |
| 42 | + | } | |
| 43 | + | usage := usageFromEvents(events) | |
| 44 | + | if usage.OutputTokens != 9 || usage.ReasoningTokens != 6 || !usage.ReasoningTokensKnown { | |
| 45 | + | t.Fatalf("usage = %+v", usage) | |
| 46 | + | } | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | func usageFromEvents(events <-chan Event) Usage { | |
| 50 | + | var usage Usage | |
| 51 | + | for event := range events { | |
| 52 | + | if e, ok := event.(EventUsage); ok { | |
| 53 | + | usage = e.Usage | |
| 54 | + | } | |
| 55 | + | } | |
| 56 | + | return usage | |
| 57 | + | } | |
| @@ -0,0 +1,57 @@ | |||||
| 1 | + | package provider | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "fmt" | |||
| 6 | + | "net/http" | |||
| 7 | + | "net/http/httptest" | |||
| 8 | + | "testing" | |||
| 9 | + | ) | |||
| 10 | + | ||||
| 11 | + | func TestOpenAIReportsReasoningTokens(t *testing.T) { | |||
| 12 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |||
| 13 | + | w.Header().Set("content-type", "text/event-stream") | |||
| 14 | + | fmt.Fprintln(w, `data: {"choices":[{"delta":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":12,"completion_tokens":9,"completion_tokens_details":{"reasoning_tokens":6}}}`) | |||
| 15 | + | fmt.Fprintln(w) | |||
| 16 | + | fmt.Fprintln(w, "data: [DONE]") | |||
| 17 | + | fmt.Fprintln(w) | |||
| 18 | + | })) | |||
| 19 | + | defer srv.Close() | |||
| 20 | + | ||||
| 21 | + | events, err := NewOpenAI("test", srv.URL).Stream(context.Background(), Request{Model: "gpt-test"}) | |||
| 22 | + | if err != nil { | |||
| 23 | + | t.Fatal(err) | |||
| 24 | + | } | |||
| 25 | + | usage := usageFromEvents(events) | |||
| 26 | + | if usage.OutputTokens != 9 || usage.ReasoningTokens != 6 || !usage.ReasoningTokensKnown { | |||
| 27 | + | t.Fatalf("usage = %+v", usage) | |||
| 28 | + | } | |||
| 29 | + | } | |||
| 30 | + | ||||
| 31 | + | func TestOpenAIResponsesReportsReasoningTokens(t *testing.T) { | |||
| 32 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |||
| 33 | + | w.Header().Set("content-type", "text/event-stream") | |||
| 34 | + | fmt.Fprintln(w, `data: {"type":"response.completed","response":{"status":"completed","usage":{"input_tokens":12,"output_tokens":9,"output_tokens_details":{"reasoning_tokens":6}}}}`) | |||
| 35 | + | fmt.Fprintln(w) | |||
| 36 | + | })) | |||
| 37 | + | defer srv.Close() | |||
| 38 | + | ||||
| 39 | + | events, err := NewOpenAICodex("test", "account", srv.URL).Stream(context.Background(), Request{Model: "gpt-5.5"}) | |||
| 40 | + | if err != nil { | |||
| 41 | + | t.Fatal(err) | |||
| 42 | + | } | |||
| 43 | + | usage := usageFromEvents(events) | |||
| 44 | + | if usage.OutputTokens != 9 || usage.ReasoningTokens != 6 || !usage.ReasoningTokensKnown { | |||
| 45 | + | t.Fatalf("usage = %+v", usage) | |||
| 46 | + | } | |||
| 47 | + | } | |||
| 48 | + | ||||
| 49 | + | func usageFromEvents(events <-chan Event) Usage { | |||
| 50 | + | var usage Usage | |||
| 51 | + | for event := range events { | |||
| 52 | + | if e, ok := event.(EventUsage); ok { | |||
| 53 | + | usage = e.Usage | |||
| 54 | + | } | |||
| 55 | + | } | |||
| 56 | + | return usage | |||
| 57 | + | } | |||