Fix OpenAI-compatible tool result images
- commit
- 80dd4c69cc42fa52a11c00ed7a9c18af08668dfc
- author
- patriceckhart · · 2026-09-08 05:40 UTC
- parent
- 94efc28
- signature
- unsigned
- refs
- tag: v0.3.67
2 files · +174 -30
M packages/provider/openai.go +75 -30
| @@ -303,7 +303,9 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||
| 303 | 303 | textOnly := c.name == "deepseek" | |
| 304 | 304 | ||
| 305 | 305 | req.Messages = RepairOrphanedToolResults(req.Messages) | |
| 306 | - | for _, msg := range req.Messages { | |
| 306 | + | for msgIndex := 0; msgIndex < len(req.Messages); msgIndex++ { | |
| 307 | + | msg := req.Messages[msgIndex] | |
| 308 | + | addedToolNames := append([]string(nil), msg.AddedToolNames...) | |
| 307 | 309 | switch msg.Role { | |
| 308 | 310 | case RoleUser: | |
| 309 | 311 | content := buildOAIUserContent(msg.Content, textOnly) | |
| @@ -363,23 +365,53 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||
| 363 | 365 | } | |
| 364 | 366 | out.Messages = append(out.Messages, am) | |
| 365 | 367 | case RoleTool: | |
| 366 | - | // Each ToolResultBlock becomes its own tool message. Preserve | |
| 367 | - | // image blocks for vision-capable OpenAI models instead of | |
| 368 | - | // flattening the tool output to plain text. | |
| 369 | - | for _, b := range msg.Content { | |
| 370 | - | if tr, ok := b.(ToolResultBlock); ok { | |
| 371 | - | content := buildOAIToolContent(tr.Content, tr.IsError, textOnly) | |
| 372 | - | out.Messages = append(out.Messages, oaiMessage{ | |
| 373 | - | Role: "tool", | |
| 374 | - | ToolCallID: tr.CallID, | |
| 375 | - | Content: content, | |
| 376 | - | }) | |
| 368 | + | // Chat Completions tool messages only carry text. Keep every | |
| 369 | + | // tool result paired with its call, then deliver any images in | |
| 370 | + | // one user message after the complete batch of tool results. | |
| 371 | + | // Strict OpenAI-compatible APIs reject image_url blocks on a | |
| 372 | + | // role=tool message. | |
| 373 | + | var images []Content | |
| 374 | + | groupEnd := msgIndex | |
| 375 | + | for groupEnd < len(req.Messages) && req.Messages[groupEnd].Role == RoleTool { | |
| 376 | + | toolMessage := req.Messages[groupEnd] | |
| 377 | + | if groupEnd > msgIndex { | |
| 378 | + | addedToolNames = append(addedToolNames, toolMessage.AddedToolNames...) | |
| 379 | + | } | |
| 380 | + | for _, b := range toolMessage.Content { | |
| 381 | + | if tr, ok := b.(ToolResultBlock); ok { | |
| 382 | + | content := buildOAIToolContent(tr.Content, tr.IsError, textOnly) | |
| 383 | + | out.Messages = append(out.Messages, oaiMessage{ | |
| 384 | + | Role: "tool", | |
| 385 | + | ToolCallID: tr.CallID, | |
| 386 | + | Content: content, | |
| 387 | + | }) | |
| 388 | + | if !textOnly { | |
| 389 | + | for _, inner := range tr.Content { | |
| 390 | + | if image, ok := inner.(ImageBlock); ok { | |
| 391 | + | images = append(images, image) | |
| 392 | + | } | |
| 393 | + | } | |
| 394 | + | } | |
| 395 | + | } | |
| 377 | 396 | } | |
| 397 | + | groupEnd++ | |
| 398 | + | } | |
| 399 | + | // The built-in OpenAI client also persists this image mirror so | |
| 400 | + | // Responses-routed models can consume it. Reuse that next message | |
| 401 | + | // instead of sending the image twice. | |
| 402 | + | hasPersistedMirror := groupEnd < len(req.Messages) && isOpenAIToolImageMirror(req.Messages[groupEnd]) | |
| 403 | + | if len(images) > 0 && !hasPersistedMirror { | |
| 404 | + | content := append([]Content{TextBlock{Text: "Tool output included the following image content:"}}, images...) | |
| 405 | + | out.Messages = append(out.Messages, oaiMessage{ | |
| 406 | + | Role: "user", | |
| 407 | + | Content: buildOAIContentBlocks(content, false), | |
| 408 | + | }) | |
| 378 | 409 | } | |
| 410 | + | msgIndex = groupEnd - 1 | |
| 379 | 411 | } | |
| 380 | 412 | if deferredMode { | |
| 381 | 413 | var loaded []oaiTool | |
| 382 | - | for _, name := range msg.AddedToolNames { | |
| 414 | + | for _, name := range addedToolNames { | |
| 383 | 415 | if t, ok := toolByName[name]; ok && t.Deferred { | |
| 384 | 416 | loaded = append(loaded, makeOAITool(t)) | |
| 385 | 417 | } | |
| @@ -468,30 +500,43 @@ func buildOAIUserContent(blocks []Content, textOnly bool) interface{} { | |||
| 468 | 500 | return buildOAIContentBlocks(blocks, false) | |
| 469 | 501 | } | |
| 470 | 502 | ||
| 471 | - | func buildOAIToolContent(blocks []Content, isError, textOnly bool) interface{} { | |
| 503 | + | func buildOAIToolContent(blocks []Content, isError, textOnly bool) string { | |
| 504 | + | var sb strings.Builder | |
| 472 | 505 | hasImage := false | |
| 473 | 506 | for _, b := range blocks { | |
| 474 | - | if _, ok := b.(ImageBlock); ok { | |
| 507 | + | switch v := b.(type) { | |
| 508 | + | case TextBlock: | |
| 509 | + | if sb.Len() > 0 { | |
| 510 | + | sb.WriteString("\n") | |
| 511 | + | } | |
| 512 | + | sb.WriteString(v.Text) | |
| 513 | + | case ImageBlock: | |
| 475 | 514 | hasImage = true | |
| 476 | - | break | |
| 477 | 515 | } | |
| 478 | 516 | } | |
| 479 | - | if textOnly || !hasImage { | |
| 480 | - | var sb strings.Builder | |
| 481 | - | for _, b := range blocks { | |
| 482 | - | if tb, ok := b.(TextBlock); ok { | |
| 483 | - | if sb.Len() > 0 { | |
| 484 | - | sb.WriteString("\n") | |
| 485 | - | } | |
| 486 | - | sb.WriteString(tb.Text) | |
| 487 | - | } | |
| 488 | - | } | |
| 489 | - | if isError && sb.Len() > 0 { | |
| 490 | - | sb.WriteString(" [error]") | |
| 517 | + | if sb.Len() == 0 && hasImage && !textOnly { | |
| 518 | + | sb.WriteString("(see attached image)") | |
| 519 | + | } | |
| 520 | + | if isError && sb.Len() > 0 { | |
| 521 | + | sb.WriteString(" [error]") | |
| 522 | + | } | |
| 523 | + | return sb.String() | |
| 524 | + | } | |
| 525 | + | ||
| 526 | + | func isOpenAIToolImageMirror(msg Message) bool { | |
| 527 | + | if msg.Role != RoleUser || len(msg.Content) < 2 { | |
| 528 | + | return false | |
| 529 | + | } | |
| 530 | + | text, ok := msg.Content[0].(TextBlock) | |
| 531 | + | if !ok || text.Text != "Tool output included the following image content:" { | |
| 532 | + | return false | |
| 533 | + | } | |
| 534 | + | for _, content := range msg.Content[1:] { | |
| 535 | + | if _, ok := content.(ImageBlock); ok { | |
| 536 | + | return true | |
| 491 | 537 | } | |
| 492 | - | return sb.String() | |
| 493 | 538 | } | |
| 494 | - | return buildOAIContentBlocks(blocks, isError) | |
| 539 | + | return false | |
| 495 | 540 | } | |
| 496 | 541 | ||
| 497 | 542 | func buildOAIContentBlocks(blocks []Content, isError bool) []interface{} { | |
| @@ -303,7 +303,9 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||||
| 303 | textOnly := c.name == "deepseek" | 303 | textOnly := c.name == "deepseek" | ||
| 304 | 304 | ||||
| 305 | req.Messages = RepairOrphanedToolResults(req.Messages) | 305 | req.Messages = RepairOrphanedToolResults(req.Messages) | ||
| 306 | - | for _, msg := range req.Messages { | 306 | + | for msgIndex := 0; msgIndex < len(req.Messages); msgIndex++ { |
| 307 | + | msg := req.Messages[msgIndex] | |||
| 308 | + | addedToolNames := append([]string(nil), msg.AddedToolNames...) | |||
| 307 | switch msg.Role { | 309 | switch msg.Role { | ||
| 308 | case RoleUser: | 310 | case RoleUser: | ||
| 309 | content := buildOAIUserContent(msg.Content, textOnly) | 311 | content := buildOAIUserContent(msg.Content, textOnly) | ||
| @@ -363,23 +365,53 @@ func (c *openaiClient) buildRequest(req Request) (*oaiRequest, error) { | |||||
| 363 | } | 365 | } | ||
| 364 | out.Messages = append(out.Messages, am) | 366 | out.Messages = append(out.Messages, am) | ||
| 365 | case RoleTool: | 367 | case RoleTool: | ||
| 366 | - | // Each ToolResultBlock becomes its own tool message. Preserve | 368 | + | // Chat Completions tool messages only carry text. Keep every |
| 367 | - | // image blocks for vision-capable OpenAI models instead of | 369 | + | // tool result paired with its call, then deliver any images in |
| 368 | - | // flattening the tool output to plain text. | 370 | + | // one user message after the complete batch of tool results. |
| 369 | - | for _, b := range msg.Content { | 371 | + | // Strict OpenAI-compatible APIs reject image_url blocks on a |
| 370 | - | if tr, ok := b.(ToolResultBlock); ok { | 372 | + | // role=tool message. |
| 371 | - | content := buildOAIToolContent(tr.Content, tr.IsError, textOnly) | 373 | + | var images []Content |
| 372 | - | out.Messages = append(out.Messages, oaiMessage{ | 374 | + | groupEnd := msgIndex |
| 373 | - | Role: "tool", | 375 | + | for groupEnd < len(req.Messages) && req.Messages[groupEnd].Role == RoleTool { |
| 374 | - | ToolCallID: tr.CallID, | 376 | + | toolMessage := req.Messages[groupEnd] |
| 375 | - | Content: content, | 377 | + | if groupEnd > msgIndex { |
| 376 | - | }) | 378 | + | addedToolNames = append(addedToolNames, toolMessage.AddedToolNames...) |
| 379 | + | } | |||
| 380 | + | for _, b := range toolMessage.Content { | |||
| 381 | + | if tr, ok := b.(ToolResultBlock); ok { | |||
| 382 | + | content := buildOAIToolContent(tr.Content, tr.IsError, textOnly) | |||
| 383 | + | out.Messages = append(out.Messages, oaiMessage{ | |||
| 384 | + | Role: "tool", | |||
| 385 | + | ToolCallID: tr.CallID, | |||
| 386 | + | Content: content, | |||
| 387 | + | }) | |||
| 388 | + | if !textOnly { | |||
| 389 | + | for _, inner := range tr.Content { | |||
| 390 | + | if image, ok := inner.(ImageBlock); ok { | |||
| 391 | + | images = append(images, image) | |||
| 392 | + | } | |||
| 393 | + | } | |||
| 394 | + | } | |||
| 395 | + | } | |||
| 377 | } | 396 | } | ||
| 397 | + | groupEnd++ | |||
| 398 | + | } | |||
| 399 | + | // The built-in OpenAI client also persists this image mirror so | |||
| 400 | + | // Responses-routed models can consume it. Reuse that next message | |||
| 401 | + | // instead of sending the image twice. | |||
| 402 | + | hasPersistedMirror := groupEnd < len(req.Messages) && isOpenAIToolImageMirror(req.Messages[groupEnd]) | |||
| 403 | + | if len(images) > 0 && !hasPersistedMirror { | |||
| 404 | + | content := append([]Content{TextBlock{Text: "Tool output included the following image content:"}}, images...) | |||
| 405 | + | out.Messages = append(out.Messages, oaiMessage{ | |||
| 406 | + | Role: "user", | |||
| 407 | + | Content: buildOAIContentBlocks(content, false), | |||
| 408 | + | }) | |||
| 378 | } | 409 | } | ||
| 410 | + | msgIndex = groupEnd - 1 | |||
| 379 | } | 411 | } | ||
| 380 | if deferredMode { | 412 | if deferredMode { | ||
| 381 | var loaded []oaiTool | 413 | var loaded []oaiTool | ||
| 382 | - | for _, name := range msg.AddedToolNames { | 414 | + | for _, name := range addedToolNames { |
| 383 | if t, ok := toolByName[name]; ok && t.Deferred { | 415 | if t, ok := toolByName[name]; ok && t.Deferred { | ||
| 384 | loaded = append(loaded, makeOAITool(t)) | 416 | loaded = append(loaded, makeOAITool(t)) | ||
| 385 | } | 417 | } | ||
| @@ -468,30 +500,43 @@ func buildOAIUserContent(blocks []Content, textOnly bool) interface{} { | |||||
| 468 | return buildOAIContentBlocks(blocks, false) | 500 | return buildOAIContentBlocks(blocks, false) | ||
| 469 | } | 501 | } | ||
| 470 | 502 | ||||
| 471 | - | func buildOAIToolContent(blocks []Content, isError, textOnly bool) interface{} { | 503 | + | func buildOAIToolContent(blocks []Content, isError, textOnly bool) string { |
| 504 | + | var sb strings.Builder | |||
| 472 | hasImage := false | 505 | hasImage := false | ||
| 473 | for _, b := range blocks { | 506 | for _, b := range blocks { | ||
| 474 | - | if _, ok := b.(ImageBlock); ok { | 507 | + | switch v := b.(type) { |
| 508 | + | case TextBlock: | |||
| 509 | + | if sb.Len() > 0 { | |||
| 510 | + | sb.WriteString("\n") | |||
| 511 | + | } | |||
| 512 | + | sb.WriteString(v.Text) | |||
| 513 | + | case ImageBlock: | |||
| 475 | hasImage = true | 514 | hasImage = true | ||
| 476 | - | break | |||
| 477 | } | 515 | } | ||
| 478 | } | 516 | } | ||
| 479 | - | if textOnly || !hasImage { | 517 | + | if sb.Len() == 0 && hasImage && !textOnly { |
| 480 | - | var sb strings.Builder | 518 | + | sb.WriteString("(see attached image)") |
| 481 | - | for _, b := range blocks { | 519 | + | } |
| 482 | - | if tb, ok := b.(TextBlock); ok { | 520 | + | if isError && sb.Len() > 0 { |
| 483 | - | if sb.Len() > 0 { | 521 | + | sb.WriteString(" [error]") |
| 484 | - | sb.WriteString("\n") | 522 | + | } |
| 485 | - | } | 523 | + | return sb.String() |
| 486 | - | sb.WriteString(tb.Text) | 524 | + | } |
| 487 | - | } | 525 | + | |
| 488 | - | } | 526 | + | func isOpenAIToolImageMirror(msg Message) bool { |
| 489 | - | if isError && sb.Len() > 0 { | 527 | + | if msg.Role != RoleUser || len(msg.Content) < 2 { |
| 490 | - | sb.WriteString(" [error]") | 528 | + | return false |
| 529 | + | } | |||
| 530 | + | text, ok := msg.Content[0].(TextBlock) | |||
| 531 | + | if !ok || text.Text != "Tool output included the following image content:" { | |||
| 532 | + | return false | |||
| 533 | + | } | |||
| 534 | + | for _, content := range msg.Content[1:] { | |||
| 535 | + | if _, ok := content.(ImageBlock); ok { | |||
| 536 | + | return true | |||
| 491 | } | 537 | } | ||
| 492 | - | return sb.String() | |||
| 493 | } | 538 | } | ||
| 494 | - | return buildOAIContentBlocks(blocks, isError) | 539 | + | return false |
| 495 | } | 540 | } | ||
| 496 | 541 | ||||
| 497 | func buildOAIContentBlocks(blocks []Content, isError bool) []interface{} { | 542 | func buildOAIContentBlocks(blocks []Content, isError bool) []interface{} { | ||
A packages/provider/openai_tool_images_test.go +99 -0
| @@ -0,0 +1,99 @@ | |||
| 1 | + | package provider | |
| 2 | + | ||
| 3 | + | import "testing" | |
| 4 | + | ||
| 5 | + | func TestOpenAICompatToolResultImagesFollowTextToolMessages(t *testing.T) { | |
| 6 | + | c := NewOpenAICompat("commandcode", "token", "https://example.test/v1", "").(*openaiClient) | |
| 7 | + | ||
| 8 | + | wire, err := c.buildRequest(Request{ | |
| 9 | + | Model: "vision-model", | |
| 10 | + | Messages: []Message{ | |
| 11 | + | {Role: RoleUser, Content: []Content{TextBlock{Text: "read both images"}}}, | |
| 12 | + | {Role: RoleAssistant, Content: []Content{ | |
| 13 | + | ToolCallBlock{ID: "call-1", Name: "read"}, | |
| 14 | + | ToolCallBlock{ID: "call-2", Name: "read"}, | |
| 15 | + | }}, | |
| 16 | + | {Role: RoleTool, Content: []Content{ | |
| 17 | + | ToolResultBlock{CallID: "call-1", Content: []Content{ | |
| 18 | + | ImageBlock{MimeType: "image/png", Data: []byte("first")}, | |
| 19 | + | }}, | |
| 20 | + | }}, | |
| 21 | + | {Role: RoleTool, Content: []Content{ | |
| 22 | + | ToolResultBlock{CallID: "call-2", Content: []Content{ | |
| 23 | + | TextBlock{Text: "second image"}, | |
| 24 | + | ImageBlock{MimeType: "image/jpeg", Data: []byte("second")}, | |
| 25 | + | }}, | |
| 26 | + | }}, | |
| 27 | + | }, | |
| 28 | + | }) | |
| 29 | + | if err != nil { | |
| 30 | + | t.Fatal(err) | |
| 31 | + | } | |
| 32 | + | ||
| 33 | + | wantRoles := []string{"user", "assistant", "tool", "tool", "user"} | |
| 34 | + | if len(wire.Messages) != len(wantRoles) { | |
| 35 | + | t.Fatalf("message count = %d, want %d: %+v", len(wire.Messages), len(wantRoles), wire.Messages) | |
| 36 | + | } | |
| 37 | + | for i, want := range wantRoles { | |
| 38 | + | if got := wire.Messages[i].Role; got != want { | |
| 39 | + | t.Fatalf("message %d role = %q, want %q", i, got, want) | |
| 40 | + | } | |
| 41 | + | } | |
| 42 | + | if got, ok := wire.Messages[2].Content.(string); !ok || got != "(see attached image)" { | |
| 43 | + | t.Fatalf("first tool content = %#v, want text placeholder", wire.Messages[2].Content) | |
| 44 | + | } | |
| 45 | + | if got, ok := wire.Messages[3].Content.(string); !ok || got != "second image" { | |
| 46 | + | t.Fatalf("second tool content = %#v, want text", wire.Messages[3].Content) | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | parts, ok := wire.Messages[4].Content.([]interface{}) | |
| 50 | + | if !ok { | |
| 51 | + | t.Fatalf("image message content type = %T, want []interface{}", wire.Messages[4].Content) | |
| 52 | + | } | |
| 53 | + | if len(parts) != 3 { | |
| 54 | + | t.Fatalf("image message parts = %d, want prefix and two images", len(parts)) | |
| 55 | + | } | |
| 56 | + | if text, ok := parts[0].(oaiContentText); !ok || text.Text != "Tool output included the following image content:" { | |
| 57 | + | t.Fatalf("image message prefix = %#v", parts[0]) | |
| 58 | + | } | |
| 59 | + | for i, part := range parts[1:] { | |
| 60 | + | image, ok := part.(oaiContentImage) | |
| 61 | + | if !ok || image.Type != "image_url" { | |
| 62 | + | t.Fatalf("image part %d = %#v", i, part) | |
| 63 | + | } | |
| 64 | + | } | |
| 65 | + | } | |
| 66 | + | ||
| 67 | + | func TestOpenAICompatDoesNotDuplicatePersistedToolImageMirror(t *testing.T) { | |
| 68 | + | c := NewOpenAICompat("custom", "token", "https://example.test/v1", "").(*openaiClient) | |
| 69 | + | image := ImageBlock{MimeType: "image/png", Data: []byte("image")} | |
| 70 | + | ||
| 71 | + | wire, err := c.buildRequest(Request{ | |
| 72 | + | Model: "vision-model", | |
| 73 | + | Messages: []Message{ | |
| 74 | + | {Role: RoleAssistant, Content: []Content{ | |
| 75 | + | ToolCallBlock{ID: "call-1", Name: "read"}, | |
| 76 | + | }}, | |
| 77 | + | {Role: RoleTool, Content: []Content{ | |
| 78 | + | ToolResultBlock{CallID: "call-1", Content: []Content{image}}, | |
| 79 | + | }}, | |
| 80 | + | {Role: RoleUser, Content: []Content{ | |
| 81 | + | TextBlock{Text: "Tool output included the following image content:"}, | |
| 82 | + | image, | |
| 83 | + | }}, | |
| 84 | + | }, | |
| 85 | + | }) | |
| 86 | + | if err != nil { | |
| 87 | + | t.Fatal(err) | |
| 88 | + | } | |
| 89 | + | ||
| 90 | + | wantRoles := []string{"assistant", "tool", "user"} | |
| 91 | + | if len(wire.Messages) != len(wantRoles) { | |
| 92 | + | t.Fatalf("message count = %d, want %d: %+v", len(wire.Messages), len(wantRoles), wire.Messages) | |
| 93 | + | } | |
| 94 | + | for i, want := range wantRoles { | |
| 95 | + | if got := wire.Messages[i].Role; got != want { | |
| 96 | + | t.Fatalf("message %d role = %q, want %q", i, got, want) | |
| 97 | + | } | |
| 98 | + | } | |
| 99 | + | } | |
| @@ -0,0 +1,99 @@ | |||||
| 1 | + | package provider | |||
| 2 | + | ||||
| 3 | + | import "testing" | |||
| 4 | + | ||||
| 5 | + | func TestOpenAICompatToolResultImagesFollowTextToolMessages(t *testing.T) { | |||
| 6 | + | c := NewOpenAICompat("commandcode", "token", "https://example.test/v1", "").(*openaiClient) | |||
| 7 | + | ||||
| 8 | + | wire, err := c.buildRequest(Request{ | |||
| 9 | + | Model: "vision-model", | |||
| 10 | + | Messages: []Message{ | |||
| 11 | + | {Role: RoleUser, Content: []Content{TextBlock{Text: "read both images"}}}, | |||
| 12 | + | {Role: RoleAssistant, Content: []Content{ | |||
| 13 | + | ToolCallBlock{ID: "call-1", Name: "read"}, | |||
| 14 | + | ToolCallBlock{ID: "call-2", Name: "read"}, | |||
| 15 | + | }}, | |||
| 16 | + | {Role: RoleTool, Content: []Content{ | |||
| 17 | + | ToolResultBlock{CallID: "call-1", Content: []Content{ | |||
| 18 | + | ImageBlock{MimeType: "image/png", Data: []byte("first")}, | |||
| 19 | + | }}, | |||
| 20 | + | }}, | |||
| 21 | + | {Role: RoleTool, Content: []Content{ | |||
| 22 | + | ToolResultBlock{CallID: "call-2", Content: []Content{ | |||
| 23 | + | TextBlock{Text: "second image"}, | |||
| 24 | + | ImageBlock{MimeType: "image/jpeg", Data: []byte("second")}, | |||
| 25 | + | }}, | |||
| 26 | + | }}, | |||
| 27 | + | }, | |||
| 28 | + | }) | |||
| 29 | + | if err != nil { | |||
| 30 | + | t.Fatal(err) | |||
| 31 | + | } | |||
| 32 | + | ||||
| 33 | + | wantRoles := []string{"user", "assistant", "tool", "tool", "user"} | |||
| 34 | + | if len(wire.Messages) != len(wantRoles) { | |||
| 35 | + | t.Fatalf("message count = %d, want %d: %+v", len(wire.Messages), len(wantRoles), wire.Messages) | |||
| 36 | + | } | |||
| 37 | + | for i, want := range wantRoles { | |||
| 38 | + | if got := wire.Messages[i].Role; got != want { | |||
| 39 | + | t.Fatalf("message %d role = %q, want %q", i, got, want) | |||
| 40 | + | } | |||
| 41 | + | } | |||
| 42 | + | if got, ok := wire.Messages[2].Content.(string); !ok || got != "(see attached image)" { | |||
| 43 | + | t.Fatalf("first tool content = %#v, want text placeholder", wire.Messages[2].Content) | |||
| 44 | + | } | |||
| 45 | + | if got, ok := wire.Messages[3].Content.(string); !ok || got != "second image" { | |||
| 46 | + | t.Fatalf("second tool content = %#v, want text", wire.Messages[3].Content) | |||
| 47 | + | } | |||
| 48 | + | ||||
| 49 | + | parts, ok := wire.Messages[4].Content.([]interface{}) | |||
| 50 | + | if !ok { | |||
| 51 | + | t.Fatalf("image message content type = %T, want []interface{}", wire.Messages[4].Content) | |||
| 52 | + | } | |||
| 53 | + | if len(parts) != 3 { | |||
| 54 | + | t.Fatalf("image message parts = %d, want prefix and two images", len(parts)) | |||
| 55 | + | } | |||
| 56 | + | if text, ok := parts[0].(oaiContentText); !ok || text.Text != "Tool output included the following image content:" { | |||
| 57 | + | t.Fatalf("image message prefix = %#v", parts[0]) | |||
| 58 | + | } | |||
| 59 | + | for i, part := range parts[1:] { | |||
| 60 | + | image, ok := part.(oaiContentImage) | |||
| 61 | + | if !ok || image.Type != "image_url" { | |||
| 62 | + | t.Fatalf("image part %d = %#v", i, part) | |||
| 63 | + | } | |||
| 64 | + | } | |||
| 65 | + | } | |||
| 66 | + | ||||
| 67 | + | func TestOpenAICompatDoesNotDuplicatePersistedToolImageMirror(t *testing.T) { | |||
| 68 | + | c := NewOpenAICompat("custom", "token", "https://example.test/v1", "").(*openaiClient) | |||
| 69 | + | image := ImageBlock{MimeType: "image/png", Data: []byte("image")} | |||
| 70 | + | ||||
| 71 | + | wire, err := c.buildRequest(Request{ | |||
| 72 | + | Model: "vision-model", | |||
| 73 | + | Messages: []Message{ | |||
| 74 | + | {Role: RoleAssistant, Content: []Content{ | |||
| 75 | + | ToolCallBlock{ID: "call-1", Name: "read"}, | |||
| 76 | + | }}, | |||
| 77 | + | {Role: RoleTool, Content: []Content{ | |||
| 78 | + | ToolResultBlock{CallID: "call-1", Content: []Content{image}}, | |||
| 79 | + | }}, | |||
| 80 | + | {Role: RoleUser, Content: []Content{ | |||
| 81 | + | TextBlock{Text: "Tool output included the following image content:"}, | |||
| 82 | + | image, | |||
| 83 | + | }}, | |||
| 84 | + | }, | |||
| 85 | + | }) | |||
| 86 | + | if err != nil { | |||
| 87 | + | t.Fatal(err) | |||
| 88 | + | } | |||
| 89 | + | ||||
| 90 | + | wantRoles := []string{"assistant", "tool", "user"} | |||
| 91 | + | if len(wire.Messages) != len(wantRoles) { | |||
| 92 | + | t.Fatalf("message count = %d, want %d: %+v", len(wire.Messages), len(wantRoles), wire.Messages) | |||
| 93 | + | } | |||
| 94 | + | for i, want := range wantRoles { | |||
| 95 | + | if got := wire.Messages[i].Role; got != want { | |||
| 96 | + | t.Fatalf("message %d role = %q, want %q", i, got, want) | |||
| 97 | + | } | |||
| 98 | + | } | |||
| 99 | + | } | |||