anthropic: downscale oversized images on outbound requests

Anthropic returns 'At least one of the image dimensions exceed max
allowed size for many-image requests' when a request contains more
than one image and any of them is larger than 2000 px on the
longest side. Single-image requests get a more lenient 8000 px
cap. OpenAI has no such limit, so a session that worked fine on
gpt-* breaks the moment /model swaps to a Claude family while the
transcript still holds high-res screenshots.

Add anthShrinkImageBytesIfTooBig that decodes any outbound image
exceeding 2000 px on the longest side, resamples it with Catmull-
Rom from golang.org/x/image/draw, and re-encodes in the same
format. JPEG stays JPEG, PNG stays PNG, GIF gets re-encoded as
PNG (we'd otherwise need to compose a single-frame gif by hand).
Decode/encode failures fall back to the original bytes so the
caller's flow is untouched and Anthropic's own error message
surfaces if the image is genuinely unusable.

Both Anthropic encode sites \u2014 the top-level message converter
and the tool-result inner converter \u2014 run every image through
the helper, so screenshots from earlier turns also get resized
on subsequent requests instead of failing forever once the
transcript captured them at full size.

Adds golang.org/x/image v0.18.0 (compatible with go 1.22).

commit: ef7327ca7d6dc69f0c89167ca7ca959795e87277
author: patriceckhart · 5mo ago · 2026-04-28 10:48 UTC
parent: 38717f5
signature: unsigned
refs: tag: v0.1.37

5 files · +225 -4

M go.mod +1 -0


@@ -13,4 +13,5 @@ require (
13	13		require (
14	14		github.com/dlclark/regexp2 v1.11.5 // indirect
15	15		github.com/rivo/uniseg v0.2.0 // indirect
	16	+	golang.org/x/image v0.18.0 // indirect
16	17		)


@@ -13,4 +13,5 @@ require (
13	require (	13		require (
14	github.com/dlclark/regexp2 v1.11.5 // indirect	14		github.com/dlclark/regexp2 v1.11.5 // indirect
15	github.com/rivo/uniseg v0.2.0 // indirect	15		github.com/rivo/uniseg v0.2.0 // indirect
		16	+	golang.org/x/image v0.18.0 // indirect
16	)	17		)

M go.sum +2 -0 generated file: collapsed

load diff

M internal/provider/anthropic.go +6 -4


@@ -352,12 +352,13 @@ func convertAnthContent(blocks []Content, renameTools bool) []interface{} {
352	352		}
353	353		out = append(out, anthTextBlock{Type: "text", Text: v.Text})
354	354		case ImageBlock:
	355	+	data, mime := anthShrinkImageBytesIfTooBig(v.Data, v.MimeType)
355	356		out = append(out, anthImageBlock{
356	357		Type: "image",
357	358		Source: anthImageSource{
358	359		Type: "base64",
359		-	MediaType: v.MimeType,
360		-	Data: base64.StdEncoding.EncodeToString(v.Data),
	360	+	MediaType: mime,
	361	+	Data: base64.StdEncoding.EncodeToString(data),
361	362		},
362	363		})
363	364		case ToolCallBlock:

@@ -413,12 +414,13 @@ func anthBuildToolResultContent(blocks []Content) (json.RawMessage, error) {
413	414		case TextBlock:
414	415		arr = append(arr, anthTextBlock{Type: "text", Text: v.Text})
415	416		case ImageBlock:
	417	+	data, mime := anthShrinkImageBytesIfTooBig(v.Data, v.MimeType)
416	418		arr = append(arr, anthImageBlock{
417	419		Type: "image",
418	420		Source: anthImageSource{
419	421		Type: "base64",
420		-	MediaType: v.MimeType,
421		-	Data: base64.StdEncoding.EncodeToString(v.Data),
	422	+	MediaType: mime,
	423	+	Data: base64.StdEncoding.EncodeToString(data),
422	424		},
423	425		})
424	426		}

 			}
 			out = append(out, anthTextBlock{Type: "text", Text: v.Text})
 		case ImageBlock:
+			data, mime := anthShrinkImageBytesIfTooBig(v.Data, v.MimeType)
 			out = append(out, anthImageBlock{
 				Type: "image",
 				Source: anthImageSource{
 					Type:      "base64",
-					MediaType: v.MimeType,
+					MediaType: mime,
-					Data:      base64.StdEncoding.EncodeToString(v.Data),
+					Data:      base64.StdEncoding.EncodeToString(data),
 				},
 			})
 		case ToolCallBlock:
 		case TextBlock:
 			arr = append(arr, anthTextBlock{Type: "text", Text: v.Text})
 		case ImageBlock:
+			data, mime := anthShrinkImageBytesIfTooBig(v.Data, v.MimeType)
 			arr = append(arr, anthImageBlock{
 				Type: "image",
 				Source: anthImageSource{
 					Type:      "base64",
-					MediaType: v.MimeType,
+					MediaType: mime,
-					Data:      base64.StdEncoding.EncodeToString(v.Data),
+					Data:      base64.StdEncoding.EncodeToString(data),
 				},
 			})
 		}

A internal/provider/anthropic_image.go +100 -0

@@ -0,0 +1,100 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"image"
	6	+	"image/jpeg"
	7	+	"image/png"
	8	+	"strings"
	9	+
	10	+	xdraw "golang.org/x/image/draw"
	11	+
	12	+	// Register decoders for the formats we want to accept on input.
	13	+	_ "image/gif"
	14	+	_ "image/jpeg"
	15	+	_ "image/png"
	16	+	)
	17	+
	18	+	// anthMaxImageSide is Anthropic's per-image dimension cap when a
	19	+	// request contains more than one image. Single-image requests are
	20	+	// allowed up to 8000 px, but zot conversations routinely include
	21	+	// multiple images (a screenshot here, a tool result image there)
	22	+	// so we always normalise down to the stricter limit. 2000 px on the
	23	+	// longest side is well below their cap and still readable for the
	24	+	// kinds of screenshots / charts the model usually consumes.
	25	+	const anthMaxImageSide = 2000
	26	+
	27	+	// anthShrinkImageBytesIfTooBig returns data unchanged when the image
	28	+	// already fits within Anthropic's per-image dimension cap. When it
	29	+	// doesn't, the image is decoded, resampled with Catmull-Rom (a good
	30	+	// balance of quality and speed for downscaling), and re-encoded in
	31	+	// the same format. The MIME type may be rewritten when re-encoding
	32	+	// requires a format change (e.g. an image originally tagged image/gif
	33	+	// is shipped as image/png after resize, since stdlib only encodes
	34	+	// gif at the package's own narrow API which is awkward for a single
	35	+	// frame).
	36	+	//
	37	+	// Errors during decode/encode return the original bytes untouched so
	38	+	// the caller's existing flow continues to work; Anthropic will then
	39	+	// emit its own clearer error than we could synthesise. We log no
	40	+	// failures here because this runs on every outbound request and is
	41	+	// best-effort.
	42	+	func anthShrinkImageBytesIfTooBig(data []byte, mime string) ([]byte, string) {
	43	+	if len(data) == 0 {
	44	+	return data, mime
	45	+	}
	46	+	cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
	47	+	if err != nil {
	48	+	return data, mime
	49	+	}
	50	+	if cfg.Width <= anthMaxImageSide && cfg.Height <= anthMaxImageSide {
	51	+	return data, mime
	52	+	}
	53	+	// Compute target dimensions preserving aspect ratio, longest side
	54	+	// clamped to anthMaxImageSide.
	55	+	tw, th := cfg.Width, cfg.Height
	56	+	if tw >= th {
	57	+	th = th * anthMaxImageSide / tw
	58	+	tw = anthMaxImageSide
	59	+	} else {
	60	+	tw = tw * anthMaxImageSide / th
	61	+	th = anthMaxImageSide
	62	+	}
	63	+	if tw < 1 {
	64	+	tw = 1
	65	+	}
	66	+	if th < 1 {
	67	+	th = 1
	68	+	}
	69	+
	70	+	src, _, err := image.Decode(bytes.NewReader(data))
	71	+	if err != nil {
	72	+	return data, mime
	73	+	}
	74	+	dst := image.NewRGBA(image.Rect(0, 0, tw, th))
	75	+	xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), xdraw.Over, nil)
	76	+
	77	+	var buf bytes.Buffer
	78	+	switch strings.ToLower(format) {
	79	+	case "jpeg":
	80	+	if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 85}); err != nil {
	81	+	return data, mime
	82	+	}
	83	+	return buf.Bytes(), "image/jpeg"
	84	+	case "png", "gif":
	85	+	// gifs are re-encoded as png so the call site stays simple
	86	+	// (avoiding a single-frame Encode helper). Anthropic accepts
	87	+	// image/png happily; the visual result is the same for the
	88	+	// model's vision pipeline.
	89	+	if err := png.Encode(&buf, dst); err != nil {
	90	+	return data, mime
	91	+	}
	92	+	nextMime := mime
	93	+	if strings.ToLower(format) == "gif" {
	94	+	nextMime = "image/png"
	95	+	}
	96	+	return buf.Bytes(), nextMime
	97	+	default:
	98	+	return data, mime
	99	+	}
	100	+	}

@@ -0,0 +1,100 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"image"
	6	+	"image/jpeg"
	7	+	"image/png"
	8	+	"strings"
	9	+
	10	+	xdraw "golang.org/x/image/draw"
	11	+
	12	+	// Register decoders for the formats we want to accept on input.
	13	+	_ "image/gif"
	14	+	_ "image/jpeg"
	15	+	_ "image/png"
	16	+	)
	17	+
	18	+	// anthMaxImageSide is Anthropic's per-image dimension cap when a
	19	+	// request contains more than one image. Single-image requests are
	20	+	// allowed up to 8000 px, but zot conversations routinely include
	21	+	// multiple images (a screenshot here, a tool result image there)
	22	+	// so we always normalise down to the stricter limit. 2000 px on the
	23	+	// longest side is well below their cap and still readable for the
	24	+	// kinds of screenshots / charts the model usually consumes.
	25	+	const anthMaxImageSide = 2000
	26	+
	27	+	// anthShrinkImageBytesIfTooBig returns data unchanged when the image
	28	+	// already fits within Anthropic's per-image dimension cap. When it
	29	+	// doesn't, the image is decoded, resampled with Catmull-Rom (a good
	30	+	// balance of quality and speed for downscaling), and re-encoded in
	31	+	// the same format. The MIME type may be rewritten when re-encoding
	32	+	// requires a format change (e.g. an image originally tagged image/gif
	33	+	// is shipped as image/png after resize, since stdlib only encodes
	34	+	// gif at the package's own narrow API which is awkward for a single
	35	+	// frame).
	36	+	//
	37	+	// Errors during decode/encode return the original bytes untouched so
	38	+	// the caller's existing flow continues to work; Anthropic will then
	39	+	// emit its own clearer error than we could synthesise. We log no
	40	+	// failures here because this runs on every outbound request and is
	41	+	// best-effort.
	42	+	func anthShrinkImageBytesIfTooBig(data []byte, mime string) ([]byte, string) {
	43	+	if len(data) == 0 {
	44	+	return data, mime
	45	+	}
	46	+	cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
	47	+	if err != nil {
	48	+	return data, mime
	49	+	}
	50	+	if cfg.Width <= anthMaxImageSide && cfg.Height <= anthMaxImageSide {
	51	+	return data, mime
	52	+	}
	53	+	// Compute target dimensions preserving aspect ratio, longest side
	54	+	// clamped to anthMaxImageSide.
	55	+	tw, th := cfg.Width, cfg.Height
	56	+	if tw >= th {
	57	+	th = th * anthMaxImageSide / tw
	58	+	tw = anthMaxImageSide
	59	+	} else {
	60	+	tw = tw * anthMaxImageSide / th
	61	+	th = anthMaxImageSide
	62	+	}
	63	+	if tw < 1 {
	64	+	tw = 1
	65	+	}
	66	+	if th < 1 {
	67	+	th = 1
	68	+	}
	69	+
	70	+	src, _, err := image.Decode(bytes.NewReader(data))
	71	+	if err != nil {
	72	+	return data, mime
	73	+	}
	74	+	dst := image.NewRGBA(image.Rect(0, 0, tw, th))
	75	+	xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), xdraw.Over, nil)
	76	+
	77	+	var buf bytes.Buffer
	78	+	switch strings.ToLower(format) {
	79	+	case "jpeg":
	80	+	if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 85}); err != nil {
	81	+	return data, mime
	82	+	}
	83	+	return buf.Bytes(), "image/jpeg"
	84	+	case "png", "gif":
	85	+	// gifs are re-encoded as png so the call site stays simple
	86	+	// (avoiding a single-frame Encode helper). Anthropic accepts
	87	+	// image/png happily; the visual result is the same for the
	88	+	// model's vision pipeline.
	89	+	if err := png.Encode(&buf, dst); err != nil {
	90	+	return data, mime
	91	+	}
	92	+	nextMime := mime
	93	+	if strings.ToLower(format) == "gif" {
	94	+	nextMime = "image/png"
	95	+	}
	96	+	return buf.Bytes(), nextMime
	97	+	default:
	98	+	return data, mime
	99	+	}
	100	+	}

A internal/provider/anthropic_image_test.go +116 -0

@@ -0,0 +1,116 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"image"
	6	+	"image/color"
	7	+	"image/jpeg"
	8	+	"image/png"
	9	+	"testing"
	10	+	)
	11	+
	12	+	func makeRect(w, h int) image.Image {
	13	+	img := image.NewRGBA(image.Rect(0, 0, w, h))
	14	+	// Fill with a recognisable colour so a buggy resize that goes
	15	+	// transparent or zero-size is obvious.
	16	+	c := color.RGBA{R: 80, G: 200, B: 120, A: 255}
	17	+	for y := 0; y < h; y++ {
	18	+	for x := 0; x < w; x++ {
	19	+	img.Set(x, y, c)
	20	+	}
	21	+	}
	22	+	return img
	23	+	}
	24	+
	25	+	func encodePNG(t *testing.T, img image.Image) []byte {
	26	+	t.Helper()
	27	+	var buf bytes.Buffer
	28	+	if err := png.Encode(&buf, img); err != nil {
	29	+	t.Fatalf("encode png: %v", err)
	30	+	}
	31	+	return buf.Bytes()
	32	+	}
	33	+
	34	+	func encodeJPEG(t *testing.T, img image.Image) []byte {
	35	+	t.Helper()
	36	+	var buf bytes.Buffer
	37	+	if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil {
	38	+	t.Fatalf("encode jpeg: %v", err)
	39	+	}
	40	+	return buf.Bytes()
	41	+	}
	42	+
	43	+	func decodeConfig(t *testing.T, data []byte) image.Config {
	44	+	t.Helper()
	45	+	cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
	46	+	if err != nil {
	47	+	t.Fatalf("decode config: %v", err)
	48	+	}
	49	+	return cfg
	50	+	}
	51	+
	52	+	func TestAnthShrinkImage_PassesThroughWhenSmall(t *testing.T) {
	53	+	src := encodePNG(t, makeRect(800, 600))
	54	+	out, mime := anthShrinkImageBytesIfTooBig(src, "image/png")
	55	+	if !bytes.Equal(out, src) {
	56	+	t.Errorf("small image was rewritten; expected pass-through")
	57	+	}
	58	+	if mime != "image/png" {
	59	+	t.Errorf("mime changed unexpectedly: %s", mime)
	60	+	}
	61	+	}
	62	+
	63	+	func TestAnthShrinkImage_DownscalesWhenTooWide(t *testing.T) {
	64	+	src := encodePNG(t, makeRect(4000, 1000))
	65	+	out, mime := anthShrinkImageBytesIfTooBig(src, "image/png")
	66	+	if bytes.Equal(out, src) {
	67	+	t.Fatalf("image was not resized")
	68	+	}
	69	+	if mime != "image/png" {
	70	+	t.Errorf("mime changed: %s", mime)
	71	+	}
	72	+	cfg := decodeConfig(t, out)
	73	+	if cfg.Width != anthMaxImageSide {
	74	+	t.Errorf("width: got %d want %d", cfg.Width, anthMaxImageSide)
	75	+	}
	76	+	// Aspect ratio preserved: 4000:1000 -> 2000:500.
	77	+	if cfg.Height != 500 {
	78	+	t.Errorf("height: got %d want 500", cfg.Height)
	79	+	}
	80	+	}
	81	+
	82	+	func TestAnthShrinkImage_DownscalesWhenTooTall(t *testing.T) {
	83	+	src := encodePNG(t, makeRect(1500, 6000))
	84	+	out, _ := anthShrinkImageBytesIfTooBig(src, "image/png")
	85	+	cfg := decodeConfig(t, out)
	86	+	if cfg.Height != anthMaxImageSide {
	87	+	t.Errorf("height: got %d want %d", cfg.Height, anthMaxImageSide)
	88	+	}
	89	+	// 1500:6000 -> 500:2000.
	90	+	if cfg.Width != 500 {
	91	+	t.Errorf("width: got %d want 500", cfg.Width)
	92	+	}
	93	+	}
	94	+
	95	+	func TestAnthShrinkImage_PreservesJPEGFormat(t *testing.T) {
	96	+	src := encodeJPEG(t, makeRect(3000, 2500))
	97	+	out, mime := anthShrinkImageBytesIfTooBig(src, "image/jpeg")
	98	+	if mime != "image/jpeg" {
	99	+	t.Errorf("mime should stay image/jpeg, got %s", mime)
	100	+	}
	101	+	cfg := decodeConfig(t, out)
	102	+	if cfg.Width > anthMaxImageSide \|\| cfg.Height > anthMaxImageSide {
	103	+	t.Errorf("dimensions exceed cap: %dx%d", cfg.Width, cfg.Height)
	104	+	}
	105	+	}
	106	+
	107	+	func TestAnthShrinkImage_BadDataReturnsOriginal(t *testing.T) {
	108	+	src := []byte("not an image at all")
	109	+	out, mime := anthShrinkImageBytesIfTooBig(src, "image/png")
	110	+	if !bytes.Equal(out, src) {
	111	+	t.Errorf("garbage input was mutated")
	112	+	}
	113	+	if mime != "image/png" {
	114	+	t.Errorf("mime was changed on bad input: %s", mime)
	115	+	}
	116	+	}

@@ -0,0 +1,116 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"image"
	6	+	"image/color"
	7	+	"image/jpeg"
	8	+	"image/png"
	9	+	"testing"
	10	+	)
	11	+
	12	+	func makeRect(w, h int) image.Image {
	13	+	img := image.NewRGBA(image.Rect(0, 0, w, h))
	14	+	// Fill with a recognisable colour so a buggy resize that goes
	15	+	// transparent or zero-size is obvious.
	16	+	c := color.RGBA{R: 80, G: 200, B: 120, A: 255}
	17	+	for y := 0; y < h; y++ {
	18	+	for x := 0; x < w; x++ {
	19	+	img.Set(x, y, c)
	20	+	}
	21	+	}
	22	+	return img
	23	+	}
	24	+
	25	+	func encodePNG(t *testing.T, img image.Image) []byte {
	26	+	t.Helper()
	27	+	var buf bytes.Buffer
	28	+	if err := png.Encode(&buf, img); err != nil {
	29	+	t.Fatalf("encode png: %v", err)
	30	+	}
	31	+	return buf.Bytes()
	32	+	}
	33	+
	34	+	func encodeJPEG(t *testing.T, img image.Image) []byte {
	35	+	t.Helper()
	36	+	var buf bytes.Buffer
	37	+	if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil {
	38	+	t.Fatalf("encode jpeg: %v", err)
	39	+	}
	40	+	return buf.Bytes()
	41	+	}
	42	+
	43	+	func decodeConfig(t *testing.T, data []byte) image.Config {
	44	+	t.Helper()
	45	+	cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
	46	+	if err != nil {
	47	+	t.Fatalf("decode config: %v", err)
	48	+	}
	49	+	return cfg
	50	+	}
	51	+
	52	+	func TestAnthShrinkImage_PassesThroughWhenSmall(t *testing.T) {
	53	+	src := encodePNG(t, makeRect(800, 600))
	54	+	out, mime := anthShrinkImageBytesIfTooBig(src, "image/png")
	55	+	if !bytes.Equal(out, src) {
	56	+	t.Errorf("small image was rewritten; expected pass-through")
	57	+	}
	58	+	if mime != "image/png" {
	59	+	t.Errorf("mime changed unexpectedly: %s", mime)
	60	+	}
	61	+	}
	62	+
	63	+	func TestAnthShrinkImage_DownscalesWhenTooWide(t *testing.T) {
	64	+	src := encodePNG(t, makeRect(4000, 1000))
	65	+	out, mime := anthShrinkImageBytesIfTooBig(src, "image/png")
	66	+	if bytes.Equal(out, src) {
	67	+	t.Fatalf("image was not resized")
	68	+	}
	69	+	if mime != "image/png" {
	70	+	t.Errorf("mime changed: %s", mime)
	71	+	}
	72	+	cfg := decodeConfig(t, out)
	73	+	if cfg.Width != anthMaxImageSide {
	74	+	t.Errorf("width: got %d want %d", cfg.Width, anthMaxImageSide)
	75	+	}
	76	+	// Aspect ratio preserved: 4000:1000 -> 2000:500.
	77	+	if cfg.Height != 500 {
	78	+	t.Errorf("height: got %d want 500", cfg.Height)
	79	+	}
	80	+	}
	81	+
	82	+	func TestAnthShrinkImage_DownscalesWhenTooTall(t *testing.T) {
	83	+	src := encodePNG(t, makeRect(1500, 6000))
	84	+	out, _ := anthShrinkImageBytesIfTooBig(src, "image/png")
	85	+	cfg := decodeConfig(t, out)
	86	+	if cfg.Height != anthMaxImageSide {
	87	+	t.Errorf("height: got %d want %d", cfg.Height, anthMaxImageSide)
	88	+	}
	89	+	// 1500:6000 -> 500:2000.
	90	+	if cfg.Width != 500 {
	91	+	t.Errorf("width: got %d want 500", cfg.Width)
	92	+	}
	93	+	}
	94	+
	95	+	func TestAnthShrinkImage_PreservesJPEGFormat(t *testing.T) {
	96	+	src := encodeJPEG(t, makeRect(3000, 2500))
	97	+	out, mime := anthShrinkImageBytesIfTooBig(src, "image/jpeg")
	98	+	if mime != "image/jpeg" {
	99	+	t.Errorf("mime should stay image/jpeg, got %s", mime)
	100	+	}
	101	+	cfg := decodeConfig(t, out)
	102	+	if cfg.Width > anthMaxImageSide \|\| cfg.Height > anthMaxImageSide {
	103	+	t.Errorf("dimensions exceed cap: %dx%d", cfg.Width, cfg.Height)
	104	+	}
	105	+	}
	106	+
	107	+	func TestAnthShrinkImage_BadDataReturnsOriginal(t *testing.T) {
	108	+	src := []byte("not an image at all")
	109	+	out, mime := anthShrinkImageBytesIfTooBig(src, "image/png")
	110	+	if !bytes.Equal(out, src) {
	111	+	t.Errorf("garbage input was mutated")
	112	+	}
	113	+	if mime != "image/png" {
	114	+	t.Errorf("mime was changed on bad input: %s", mime)
	115	+	}
	116	+	}