fix(provider): deliver tool-result images to the OpenAI Responses route

On the openai-codex (Responses API) route a tool result serialized to a
string-only function_call_output, dropping ImageBlock content, and the
agent loop's tool-image mirror only fired for provider "openai". So
images returned by read reached the TUI but never the model, which then
correctly reported it received no image content.

Extend the mirror to fire for "openai-codex" too (its client already
serializes user-message images as input_image, so the bytes arrive),
and have the codex tool-result serializer emit a short placeholder for
an image-only result instead of an empty output the API may reject.
Adds a test covering both behaviors.

commit: ecb3b022cc7c88f89a069e6e6789806ae7bf7a2b
author: patriceckhart · 4mo ago · 2026-05-29 12:21 UTC
parent: 124d679
signature: unsigned
refs: tag: v0.2.3

3 files · +92 -4

M packages/core/agent.go +7 -1


@@ -356,8 +356,14 @@ func (a *Agent) runLoop(ctx context.Context, sink func(AgentEvent)) error {
356	356		// This keeps the transcript self-contained for providers that can
357	357		// see image blocks in tool messages while making OpenAI vision
358	358		// models actually receive the image bytes.
	359	+	//
	360	+	// The OpenAI Responses route ("openai-codex") has the same
	361	+	// text-centric tool-output shape: a function_call_output only
	362	+	// carries a string, so images in a tool result never reach the
	363	+	// model. Both providers serialize images correctly when they
	364	+	// arrive as user content, so the mirror covers them both.
359	365		var imageMirror provider.Message
360		-	if a.Client != nil && a.Client.Name() == "openai" {
	366	+	if a.Client != nil && (a.Client.Name() == "openai" \|\| a.Client.Name() == "openai-codex") {
361	367		if mirror := mirrorToolImagesAsUser(toolMsg); len(mirror.Content) > 0 {
362	368		a.messages = append(a.messages, mirror)
363	369		a.rev++

 			// This keeps the transcript self-contained for providers that can
 			// see image blocks in tool messages while making OpenAI vision
 			// models actually receive the image bytes.
+			//
+			// The OpenAI Responses route ("openai-codex") has the same
+			// text-centric tool-output shape: a function_call_output only
+			// carries a string, so images in a tool result never reach the
+			// model. Both providers serialize images correctly when they
+			// arrive as user content, so the mirror covers them both.
 			var imageMirror provider.Message
-			if a.Client != nil && a.Client.Name() == "openai" {
+			if a.Client != nil && (a.Client.Name() == "openai" || a.Client.Name() == "openai-codex") {
 				if mirror := mirrorToolImagesAsUser(toolMsg); len(mirror.Content) > 0 {
 					a.messages = append(a.messages, mirror)
 					a.rev++

M packages/provider/openai_codex.go +22 -3


@@ -262,19 +262,38 @@ func (c codexClient) buildRequest(req Request) (codexRequest, error) {
262	262		for _, c := range msg.Content {
263	263		if tr, ok := c.(ToolResultBlock); ok {
264	264		var text strings.Builder
	265	+	imageCount := 0
265	266		for _, inner := range tr.Content {
266		-	if tb, ok := inner.(TextBlock); ok {
	267	+	switch v := inner.(type) {
	268	+	case TextBlock:
267	269		if text.Len() > 0 {
268	270		text.WriteString("\n")
269	271		}
270		-	text.WriteString(tb.Text)
	272	+	text.WriteString(v.Text)
	273	+	case ImageBlock:
	274	+	imageCount++
	275	+	}
	276	+	}
	277	+	// The Responses API function_call_output only carries a
	278	+	// string, so image bytes cannot ride along here. The agent
	279	+	// loop mirrors any tool-result images into the following
	280	+	// user message (which this client does serialize as
	281	+	// input_image). Leave a short text note so an image-only
	282	+	// result is not an empty output the API may reject, and so
	283	+	// the model knows the image arrives next.
	284	+	out := text.String()
	285	+	if out == "" && imageCount > 0 {
	286	+	if imageCount == 1 {
	287	+	out = "[image returned; see the following message]"
	288	+	} else {
	289	+	out = fmt.Sprintf("[%d images returned; see the following message]", imageCount)
271	290		}
272	291		}
273	292		callID, _ := splitCallID(tr.CallID)
274	293		body.Input = append(body.Input, codexFunctionCallOutput{
275	294		Type: "function_call_output",
276	295		CallID: callID,
277		-	Output: text.String(),
	296	+	Output: out,
278	297		})
279	298		}
280	299		}

 			for _, c := range msg.Content {
 				if tr, ok := c.(ToolResultBlock); ok {
 					var text strings.Builder
+					imageCount := 0
 					for _, inner := range tr.Content {
-						if tb, ok := inner.(TextBlock); ok {
+						switch v := inner.(type) {
+						case TextBlock:
 							if text.Len() > 0 {
 								text.WriteString("\n")
 							}
-							text.WriteString(tb.Text)
+							text.WriteString(v.Text)
+						case ImageBlock:
+							imageCount++
+						}
+					}
+					// The Responses API function_call_output only carries a
+					// string, so image bytes cannot ride along here. The agent
+					// loop mirrors any tool-result images into the following
+					// user message (which this client does serialize as
+					// input_image). Leave a short text note so an image-only
+					// result is not an empty output the API may reject, and so
+					// the model knows the image arrives next.
+					out := text.String()
+					if out == "" && imageCount > 0 {
+						if imageCount == 1 {
+							out = "[image returned; see the following message]"
+						} else {
+							out = fmt.Sprintf("[%d images returned; see the following message]", imageCount)
 						}
 					}
 					callID, _ := splitCallID(tr.CallID)
 					body.Input = append(body.Input, codexFunctionCallOutput{
 						Type:   "function_call_output",
 						CallID: callID,
-						Output: text.String(),
+						Output: out,
 					})
 				}
 			}

A packages/provider/openai_codex_test.go +63 -0

@@ -0,0 +1,63 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"strings"
	5	+	"testing"
	6	+	)
	7	+
	8	+	// An image-only tool result must not serialize to an empty
	9	+	// function_call_output (the Responses API may reject it) and a
	10	+	// following user-message image must serialize as input_image so the
	11	+	// model actually receives the bytes.
	12	+	func TestCodexImageToolResultMirror(t *testing.T) {
	13	+	c := NewOpenAICodex("token", "acct", "").(*codexClient)
	14	+
	15	+	wire, err := c.buildRequest(Request{
	16	+	Model: "gpt-5.5",
	17	+	Messages: []Message{
	18	+	{Role: RoleUser, Content: []Content{TextBlock{Text: "look at this"}}},
	19	+	{Role: RoleAssistant, Content: []Content{
	20	+	ToolCallBlock{ID: "call_1", Name: "read", Arguments: []byte(`{"path":"x.png"}`)},
	21	+	}},
	22	+	{Role: RoleTool, Content: []Content{
	23	+	ToolResultBlock{CallID: "call_1", Content: []Content{
	24	+	ImageBlock{MimeType: "image/png", Data: []byte("png-bytes")},
	25	+	}},
	26	+	}},
	27	+	// The agent loop appends this mirror after an image tool result.
	28	+	{Role: RoleUser, Content: []Content{
	29	+	TextBlock{Text: "Tool output included the following image content:"},
	30	+	ImageBlock{MimeType: "image/png", Data: []byte("png-bytes")},
	31	+	}},
	32	+	},
	33	+	})
	34	+	if err != nil {
	35	+	t.Fatal(err)
	36	+	}
	37	+
	38	+	var sawFnOutput, sawInputImage bool
	39	+	for _, item := range wire.Input {
	40	+	switch v := item.(type) {
	41	+	case codexFunctionCallOutput:
	42	+	sawFnOutput = true
	43	+	if strings.TrimSpace(v.Output) == "" {
	44	+	t.Fatalf("image-only tool result produced empty function_call_output")
	45	+	}
	46	+	if !strings.Contains(strings.ToLower(v.Output), "image") {
	47	+	t.Fatalf("placeholder should mention image, got %q", v.Output)
	48	+	}
	49	+	case codexInputMessage:
	50	+	for _, ct := range v.Content {
	51	+	if img, ok := ct.(codexInputImage); ok && img.Type == "input_image" {
	52	+	sawInputImage = true
	53	+	}
	54	+	}
	55	+	}
	56	+	}
	57	+	if !sawFnOutput {
	58	+	t.Fatalf("no function_call_output emitted")
	59	+	}
	60	+	if !sawInputImage {
	61	+	t.Fatalf("mirrored user image was not serialized as input_image")
	62	+	}
	63	+	}

@@ -0,0 +1,63 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"strings"
	5	+	"testing"
	6	+	)
	7	+
	8	+	// An image-only tool result must not serialize to an empty
	9	+	// function_call_output (the Responses API may reject it) and a
	10	+	// following user-message image must serialize as input_image so the
	11	+	// model actually receives the bytes.
	12	+	func TestCodexImageToolResultMirror(t *testing.T) {
	13	+	c := NewOpenAICodex("token", "acct", "").(*codexClient)
	14	+
	15	+	wire, err := c.buildRequest(Request{
	16	+	Model: "gpt-5.5",
	17	+	Messages: []Message{
	18	+	{Role: RoleUser, Content: []Content{TextBlock{Text: "look at this"}}},
	19	+	{Role: RoleAssistant, Content: []Content{
	20	+	ToolCallBlock{ID: "call_1", Name: "read", Arguments: []byte(`{"path":"x.png"}`)},
	21	+	}},
	22	+	{Role: RoleTool, Content: []Content{
	23	+	ToolResultBlock{CallID: "call_1", Content: []Content{
	24	+	ImageBlock{MimeType: "image/png", Data: []byte("png-bytes")},
	25	+	}},
	26	+	}},
	27	+	// The agent loop appends this mirror after an image tool result.
	28	+	{Role: RoleUser, Content: []Content{
	29	+	TextBlock{Text: "Tool output included the following image content:"},
	30	+	ImageBlock{MimeType: "image/png", Data: []byte("png-bytes")},
	31	+	}},
	32	+	},
	33	+	})
	34	+	if err != nil {
	35	+	t.Fatal(err)
	36	+	}
	37	+
	38	+	var sawFnOutput, sawInputImage bool
	39	+	for _, item := range wire.Input {
	40	+	switch v := item.(type) {
	41	+	case codexFunctionCallOutput:
	42	+	sawFnOutput = true
	43	+	if strings.TrimSpace(v.Output) == "" {
	44	+	t.Fatalf("image-only tool result produced empty function_call_output")
	45	+	}
	46	+	if !strings.Contains(strings.ToLower(v.Output), "image") {
	47	+	t.Fatalf("placeholder should mention image, got %q", v.Output)
	48	+	}
	49	+	case codexInputMessage:
	50	+	for _, ct := range v.Content {
	51	+	if img, ok := ct.(codexInputImage); ok && img.Type == "input_image" {
	52	+	sawInputImage = true
	53	+	}
	54	+	}
	55	+	}
	56	+	}
	57	+	if !sawFnOutput {
	58	+	t.Fatalf("no function_call_output emitted")
	59	+	}
	60	+	if !sawInputImage {
	61	+	t.Fatalf("mirrored user image was not serialized as input_image")
	62	+	}
	63	+	}