8d3b7ff fix(openai): pass image tool results to vision models · zut

M internal/agent/modes/interactive.go +28 -1


@@ -560,7 +560,7 @@ func (i *Interactive) redraw() {
560	560
561	561		cols, _ := i.cfg.Terminal.Size()
562	562		if i.agent != nil {
563		-	i.view.Messages = i.agent.Messages()
	563	+	i.view.Messages = filterHiddenTranscriptMessages(i.agent.Messages())
564	564		} else {
565	565		i.view.Messages = nil
566	566		}

@@ -896,6 +896,33 @@ func alignSliceStartToImageBlock(chat []string, start, end int) int {
896	896		return start
897	897		}
898	898
	899	+	const hiddenOpenAIImageMirrorPrefix = "Tool output included the following image content:"
	900	+
	901	+	func filterHiddenTranscriptMessages(msgs []provider.Message) []provider.Message {
	902	+	if len(msgs) == 0 {
	903	+	return nil
	904	+	}
	905	+	out := make([]provider.Message, 0, len(msgs))
	906	+	for _, m := range msgs {
	907	+	if isHiddenTranscriptMessage(m) {
	908	+	continue
	909	+	}
	910	+	out = append(out, m)
	911	+	}
	912	+	return out
	913	+	}
	914	+
	915	+	func isHiddenTranscriptMessage(m provider.Message) bool {
	916	+	if m.Role != provider.RoleUser \|\| len(m.Content) == 0 {
	917	+	return false
	918	+	}
	919	+	tb, ok := m.Content[0].(provider.TextBlock)
	920	+	if !ok {
	921	+	return false
	922	+	}
	923	+	return strings.TrimSpace(tb.Text) == hiddenOpenAIImageMirrorPrefix
	924	+	}
	925	+
899	926		func clipBottomClippedImages(lines []string) []string {
900	927		if len(lines) == 0 {
901	928		return lines


@@ -560,7 +560,7 @@ func (i *Interactive) redraw() {
560			560
561		cols, _ := i.cfg.Terminal.Size()	561		cols, _ := i.cfg.Terminal.Size()
562		if i.agent != nil {	562		if i.agent != nil {
563	-	i.view.Messages = i.agent.Messages()	563	+	i.view.Messages = filterHiddenTranscriptMessages(i.agent.Messages())
564		} else {	564		} else {
565		i.view.Messages = nil	565		i.view.Messages = nil
566		}	566		}

@@ -896,6 +896,33 @@ func alignSliceStartToImageBlock(chat []string, start, end int) int {
896		return start	896		return start
897		}	897		}
898			898
			899	+	const hiddenOpenAIImageMirrorPrefix = "Tool output included the following image content:"
			900	+
			901	+	func filterHiddenTranscriptMessages(msgs []provider.Message) []provider.Message {
			902	+	if len(msgs) == 0 {
			903	+	return nil
			904	+	}
			905	+	out := make([]provider.Message, 0, len(msgs))
			906	+	for _, m := range msgs {
			907	+	if isHiddenTranscriptMessage(m) {
			908	+	continue
			909	+	}
			910	+	out = append(out, m)
			911	+	}
			912	+	return out
			913	+	}
			914	+
			915	+	func isHiddenTranscriptMessage(m provider.Message) bool {
			916	+	if m.Role != provider.RoleUser \|\| len(m.Content) == 0 {
			917	+	return false
			918	+	}
			919	+	tb, ok := m.Content[0].(provider.TextBlock)
			920	+	if !ok {
			921	+	return false
			922	+	}
			923	+	return strings.TrimSpace(tb.Text) == hiddenOpenAIImageMirrorPrefix
			924	+	}
			925	+
899		func clipBottomClippedImages(lines []string) []string {	926		func clipBottomClippedImages(lines []string) []string {
900		if len(lines) == 0 {	927		if len(lines) == 0 {
901		return lines	928		return lines

M internal/core/agent.go +41 -0


@@ -181,6 +181,18 @@ func (a *Agent) runLoop(ctx context.Context, sink func(AgentEvent)) error {
181	181		toolMsg, hadError := a.executeTools(ctx, assistantMsg, sink)
182	182		a.mu.Lock()
183	183		a.messages = append(a.messages, toolMsg)
	184	+	// OpenAI's chat-completions tool message shape is text-centric.
	185	+	// Vision models reliably consume images when they arrive as user
	186	+	// content, so when a tool result contains images we mirror them
	187	+	// into a synthetic user message immediately after the tool result.
	188	+	// This keeps the transcript self-contained for providers that can
	189	+	// see image blocks in tool messages while making OpenAI vision
	190	+	// models actually receive the image bytes.
	191	+	if a.Client != nil && a.Client.Name() == "openai" {
	192	+	if mirror := mirrorToolImagesAsUser(toolMsg); len(mirror.Content) > 0 {
	193	+	a.messages = append(a.messages, mirror)
	194	+	}
	195	+	}
184	196		a.mu.Unlock()
185	197		// If context was cancelled during tool execution, bail out.
186	198		if err := ctx.Err(); err != nil {

@@ -388,6 +400,35 @@ func (a *Agent) runOneTool(ctx context.Context, tc provider.ToolCallBlock, sink
388	400		// extractText concatenates all TextBlock content in a message. Used
389	401		// by BeforeAssistantMessage so guards see a single string instead of
390	402		// having to walk provider.Content themselves.
	403	+	func mirrorToolImagesAsUser(msg provider.Message) provider.Message {
	404	+	var content []provider.Content
	405	+	for _, c := range msg.Content {
	406	+	tr, ok := c.(provider.ToolResultBlock)
	407	+	if !ok {
	408	+	continue
	409	+	}
	410	+	for _, inner := range tr.Content {
	411	+	switch v := inner.(type) {
	412	+	case provider.TextBlock:
	413	+	// Keep short textual context so the model understands why
	414	+	// the images appeared, but don't duplicate giant read
	415	+	// outputs verbatim.
	416	+	if len(v.Text) > 0 && len(v.Text) <= 500 {
	417	+	content = append(content, v)
	418	+	}
	419	+	case provider.ImageBlock:
	420	+	content = append(content, v)
	421	+	}
	422	+	}
	423	+	}
	424	+	if len(content) == 0 {
	425	+	return provider.Message{}
	426	+	}
	427	+	prefix := provider.TextBlock{Text: "Tool output included the following image content:"}
	428	+	content = append([]provider.Content{prefix}, content...)
	429	+	return provider.Message{Role: provider.RoleUser, Content: content, Time: time.Now()}
	430	+	}
	431	+
391	432		func extractText(msg provider.Message) string {
392	433		var out string
393	434		for _, c := range msg.Content {


@@ -181,6 +181,18 @@ func (a *Agent) runLoop(ctx context.Context, sink func(AgentEvent)) error {
181	toolMsg, hadError := a.executeTools(ctx, assistantMsg, sink)	181		toolMsg, hadError := a.executeTools(ctx, assistantMsg, sink)
182	a.mu.Lock()	182		a.mu.Lock()
183	a.messages = append(a.messages, toolMsg)	183		a.messages = append(a.messages, toolMsg)
		184	+	// OpenAI's chat-completions tool message shape is text-centric.
		185	+	// Vision models reliably consume images when they arrive as user
		186	+	// content, so when a tool result contains images we mirror them
		187	+	// into a synthetic user message immediately after the tool result.
		188	+	// This keeps the transcript self-contained for providers that can
		189	+	// see image blocks in tool messages while making OpenAI vision
		190	+	// models actually receive the image bytes.
		191	+	if a.Client != nil && a.Client.Name() == "openai" {
		192	+	if mirror := mirrorToolImagesAsUser(toolMsg); len(mirror.Content) > 0 {
		193	+	a.messages = append(a.messages, mirror)
		194	+	}
		195	+	}
184	a.mu.Unlock()	196		a.mu.Unlock()
185	// If context was cancelled during tool execution, bail out.	197		// If context was cancelled during tool execution, bail out.
186	if err := ctx.Err(); err != nil {	198		if err := ctx.Err(); err != nil {

@@ -388,6 +400,35 @@ func (a *Agent) runOneTool(ctx context.Context, tc provider.ToolCallBlock, sink
388	// extractText concatenates all TextBlock content in a message. Used	400		// extractText concatenates all TextBlock content in a message. Used
389	// by BeforeAssistantMessage so guards see a single string instead of	401		// by BeforeAssistantMessage so guards see a single string instead of
390	// having to walk provider.Content themselves.	402		// having to walk provider.Content themselves.
		403	+	func mirrorToolImagesAsUser(msg provider.Message) provider.Message {
		404	+	var content []provider.Content
		405	+	for _, c := range msg.Content {
		406	+	tr, ok := c.(provider.ToolResultBlock)
		407	+	if !ok {
		408	+	continue
		409	+	}
		410	+	for _, inner := range tr.Content {
		411	+	switch v := inner.(type) {
		412	+	case provider.TextBlock:
		413	+	// Keep short textual context so the model understands why
		414	+	// the images appeared, but don't duplicate giant read
		415	+	// outputs verbatim.
		416	+	if len(v.Text) > 0 && len(v.Text) <= 500 {
		417	+	content = append(content, v)
		418	+	}
		419	+	case provider.ImageBlock:
		420	+	content = append(content, v)
		421	+	}
		422	+	}
		423	+	}
		424	+	if len(content) == 0 {
		425	+	return provider.Message{}
		426	+	}
		427	+	prefix := provider.TextBlock{Text: "Tool output included the following image content:"}
		428	+	content = append([]provider.Content{prefix}, content...)
		429	+	return provider.Message{Role: provider.RoleUser, Content: content, Time: time.Now()}
		430	+	}
		431	+
391	func extractText(msg provider.Message) string {	432		func extractText(msg provider.Message) string {
392	var out string	433		var out string
393	for _, c := range msg.Content {	434		for _, c := range msg.Content {

M internal/provider/openai.go +38 -14


@@ -174,25 +174,16 @@ func (c openaiClient) buildRequest(req Request) (oaiRequest, error) {
174	174		}
175	175		out.Messages = append(out.Messages, am)
176	176		case RoleTool:
177		-	// Each ToolResultBlock becomes its own tool message.
	177	+	// Each ToolResultBlock becomes its own tool message. Preserve
	178	+	// image blocks for vision-capable OpenAI models instead of
	179	+	// flattening the tool output to plain text.
178	180		for _, b := range msg.Content {
179	181		if tr, ok := b.(ToolResultBlock); ok {
180		-	var text strings.Builder
181		-	for _, inner := range tr.Content {
182		-	if tb, ok := inner.(TextBlock); ok {
183		-	if text.Len() > 0 {
184		-	text.WriteString("\n")
185		-	}
186		-	text.WriteString(tb.Text)
187		-	}
188		-	}
189		-	if tr.IsError && text.Len() > 0 {
190		-	text.WriteString(" [error]")
191		-	}
	182	+	content := buildOAIToolContent(tr.Content, tr.IsError)
192	183		out.Messages = append(out.Messages, oaiMessage{
193	184		Role: "tool",
194	185		ToolCallID: tr.CallID,
195		-	Content: text.String(),
	186	+	Content: content,
196	187		})
197	188		}
198	189		}

@@ -234,6 +225,36 @@ func buildOAIUserContent(blocks []Content) interface{} {
234	225		}
235	226		return sb.String()
236	227		}
	228	+	return buildOAIContentBlocks(blocks, false)
	229	+	}
	230	+
	231	+	func buildOAIToolContent(blocks []Content, isError bool) interface{} {
	232	+	hasImage := false
	233	+	for _, b := range blocks {
	234	+	if _, ok := b.(ImageBlock); ok {
	235	+	hasImage = true
	236	+	break
	237	+	}
	238	+	}
	239	+	if !hasImage {
	240	+	var sb strings.Builder
	241	+	for _, b := range blocks {
	242	+	if tb, ok := b.(TextBlock); ok {
	243	+	if sb.Len() > 0 {
	244	+	sb.WriteString("\n")
	245	+	}
	246	+	sb.WriteString(tb.Text)
	247	+	}
	248	+	}
	249	+	if isError && sb.Len() > 0 {
	250	+	sb.WriteString(" [error]")
	251	+	}
	252	+	return sb.String()
	253	+	}
	254	+	return buildOAIContentBlocks(blocks, isError)
	255	+	}
	256	+
	257	+	func buildOAIContentBlocks(blocks []Content, isError bool) []interface{} {
237	258		var arr []interface{}
238	259		for _, b := range blocks {
239	260		switch v := b.(type) {

@@ -246,6 +267,9 @@ func buildOAIUserContent(blocks []Content) interface{} {
246	267		arr = append(arr, img)
247	268		}
248	269		}
	270	+	if isError {
	271	+	arr = append(arr, oaiContentText{Type: "text", Text: "[error]"})
	272	+	}
249	273		return arr
250	274		}
251	275

 			}
 			out.Messages = append(out.Messages, am)
 		case RoleTool:
-			// Each ToolResultBlock becomes its own tool message.
+			// Each ToolResultBlock becomes its own tool message. Preserve
+			// image blocks for vision-capable OpenAI models instead of
+			// flattening the tool output to plain text.
 			for _, b := range msg.Content {
 				if tr, ok := b.(ToolResultBlock); ok {
-					var text strings.Builder
+					content := buildOAIToolContent(tr.Content, tr.IsError)
-					for _, inner := range tr.Content {
-						if tb, ok := inner.(TextBlock); ok {
-							if text.Len() > 0 {
-								text.WriteString("\n")
-							}
-							text.WriteString(tb.Text)
-						}
-					}
-					if tr.IsError && text.Len() > 0 {
-						text.WriteString(" [error]")
-					}
 					out.Messages = append(out.Messages, oaiMessage{
 						Role:       "tool",
 						ToolCallID: tr.CallID,
-						Content:    text.String(),
+						Content:    content,
 					})
 				}
 			}
 		}
 		return sb.String()
 	}
+	return buildOAIContentBlocks(blocks, false)
+}
+func buildOAIToolContent(blocks []Content, isError bool) interface{} {
+	hasImage := false
+	for _, b := range blocks {
+		if _, ok := b.(ImageBlock); ok {
+			hasImage = true
+			break
+		}
+	}
+	if !hasImage {
+		var sb strings.Builder
+		for _, b := range blocks {
+			if tb, ok := b.(TextBlock); ok {
+				if sb.Len() > 0 {
+					sb.WriteString("\n")
+				}
+				sb.WriteString(tb.Text)
+			}
+		}
+		if isError && sb.Len() > 0 {
+			sb.WriteString(" [error]")
+		}
+		return sb.String()
+	}
+	return buildOAIContentBlocks(blocks, isError)
+}
+func buildOAIContentBlocks(blocks []Content, isError bool) []interface{} {
 	var arr []interface{}
 	for _, b := range blocks {
 		switch v := b.(type) {
 			arr = append(arr, img)
 		}
 	}
+	if isError {
+		arr = append(arr, oaiContentText{Type: "text", Text: "[error]"})
+	}
 	return arr
 }