perf(snapshot): size the inflate buffer from the compressed length

commit: 5dde9c6cf30a0db07607083feacf0b9b723cd613
author: brice · 13d ago · 2026-10-08 20:07 UTC
parent: e0f6671
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

2 files · +13 -9

M internal/snapshotcodec/object_compress.go +12 -8


@@ -15,10 +15,14 @@ import (
15	15		// deterministic for one toolchain and level, so content addressing stays
16	16		// stable between checkpoints of one build.
17	17
18		-	// inflateInitialCap bounds the up-front allocation for one payload, so a
19		-	// corrupt declared length cannot allocate more than the stream produces.
	18	+	// inflateInitialCap is the up-front allocation floor for one payload.
20	19		const inflateInitialCap = 1 << 20
21	20
	21	+	// deflateMaxRatio bounds how many bytes one compressed byte can inflate to
	22	+	// (deflate tops out near 1032:1), so a payload the stream can really produce
	23	+	// is allocated once while a corrupt declared length stays bounded.
	24	+	const deflateMaxRatio = 1032
	25	+
22	26		var objectCompressorPool = sync.Pool{New: func() any {
23	27		w, err := zlib.NewWriterLevel(io.Discard, zlib.BestSpeed)
24	28		if err != nil {

@@ -58,7 +62,7 @@ func inflateObjectPayload(compressed []byte, size uint32) ([]byte, error) {
58	62		return nil, invalid(err)
59	63		}
60	64		defer objectDecompressorPool.Put(r)
61		-	payload, err := readExact(r, int(size))
	65	+	payload, err := readExact(r, int(size), max(inflateInitialCap, len(compressed)*deflateMaxRatio))
62	66		if err != nil {
63	67		return nil, invalid(err)
64	68		}

@@ -76,11 +80,11 @@ func inflateObjectPayload(compressed []byte, size uint32) ([]byte, error) {
76	80		}
77	81
78	82		// readExact reads exactly size bytes. It doubles the buffer as data arrives,
79		-	// starting at inflateInitialCap and capping every step at size, so a corrupt
80		-	// declared length allocates at most about four times what the stream produces
81		-	// (about three times live at peak) and the returned payload has no slack.
82		-	func readExact(r io.Reader, size int) ([]byte, error) {
83		-	buf := make([]byte, min(size, inflateInitialCap))
	83	+	// starting at initial and capping every step at size, so a corrupt declared
	84	+	// length allocates at most about four times max(initial, what the stream
	85	+	// produces) and the returned payload has no slack.
	86	+	func readExact(r io.Reader, size, initial int) ([]byte, error) {
	87	+	buf := make([]byte, min(size, initial))
84	88		for n := 0; ; {
85	89		m, err := io.ReadFull(r, buf[n:])
86	90		n += m

 // deterministic for one toolchain and level, so content addressing stays
 // stable between checkpoints of one build.
-// inflateInitialCap bounds the up-front allocation for one payload, so a
+// inflateInitialCap is the up-front allocation floor for one payload.
-// corrupt declared length cannot allocate more than the stream produces.
 const inflateInitialCap = 1 << 20
+// deflateMaxRatio bounds how many bytes one compressed byte can inflate to
+// (deflate tops out near 1032:1), so a payload the stream can really produce
+// is allocated once while a corrupt declared length stays bounded.
+const deflateMaxRatio = 1032
 var objectCompressorPool = sync.Pool{New: func() any {
 	w, err := zlib.NewWriterLevel(io.Discard, zlib.BestSpeed)
 	if err != nil {
 		return nil, invalid(err)
 	}
 	defer objectDecompressorPool.Put(r)
-	payload, err := readExact(r, int(size))
+	payload, err := readExact(r, int(size), max(inflateInitialCap, len(compressed)*deflateMaxRatio))
 	if err != nil {
 		return nil, invalid(err)
 	}
 }
 // readExact reads exactly size bytes. It doubles the buffer as data arrives,
-// starting at inflateInitialCap and capping every step at size, so a corrupt
+// starting at initial and capping every step at size, so a corrupt declared
-// declared length allocates at most about four times what the stream produces
+// length allocates at most about four times max(initial, what the stream
-// (about three times live at peak) and the returned payload has no slack.
+// produces) and the returned payload has no slack.
-func readExact(r io.Reader, size int) ([]byte, error) {
+func readExact(r io.Reader, size, initial int) ([]byte, error) {
-	buf := make([]byte, min(size, inflateInitialCap))
+	buf := make([]byte, min(size, initial))
 	for n := 0; ; {
 		m, err := io.ReadFull(r, buf[n:])
 		n += m

M internal/snapshotcodec/object_compress_test.go +1 -1


@@ -79,7 +79,7 @@ func TestInflateObjectPayloadAllocation(t *testing.T) {
79	79		}{
80	80		{name: "small payload", size: 64 << 10, maxBytes: 64<<10 + 64<<10},
81	81		{name: "payload at the initial cap", size: inflateInitialCap, maxBytes: inflateInitialCap + 64<<10},
82		-	{name: "payload above the initial cap", size: 3 << 20, maxBytes: 2*(3<<20) + 64<<10},
	82	+	{name: "payload above the initial cap", size: 3 << 20, maxBytes: 3<<20 + 64<<10},
83	83		{name: "declared maximum on a small stream", size: 64 << 10, declared: maxObjectPayloadSize, maxBytes: 2*inflateInitialCap + 64<<10},
84	84		} {
85	85		t.Run(tc.name, func(t *testing.T) {


@@ -79,7 +79,7 @@ func TestInflateObjectPayloadAllocation(t *testing.T) {
79		}{	79		}{
80		{name: "small payload", size: 64 << 10, maxBytes: 64<<10 + 64<<10},	80		{name: "small payload", size: 64 << 10, maxBytes: 64<<10 + 64<<10},
81		{name: "payload at the initial cap", size: inflateInitialCap, maxBytes: inflateInitialCap + 64<<10},	81		{name: "payload at the initial cap", size: inflateInitialCap, maxBytes: inflateInitialCap + 64<<10},
82	-	{name: "payload above the initial cap", size: 3 << 20, maxBytes: 2*(3<<20) + 64<<10},	82	+	{name: "payload above the initial cap", size: 3 << 20, maxBytes: 3<<20 + 64<<10},
83		{name: "declared maximum on a small stream", size: 64 << 10, declared: maxObjectPayloadSize, maxBytes: 2*inflateInitialCap + 64<<10},	83		{name: "declared maximum on a small stream", size: 64 << 10, declared: maxObjectPayloadSize, maxBytes: 2*inflateInitialCap + 64<<10},
84		} {	84		} {
85		t.Run(tc.name, func(t *testing.T) {	85		t.Run(tc.name, func(t *testing.T) {