1fa07cc fix(vt): accept wide history rows within budgets · vev

M pkg/vt/benchmark_test.go +23 -3


@@ -8,9 +8,10 @@ import (
8	8		)
9	9
10	10		var (
11		-	benchmarkHistoryViewSink HistoryView
12		-	benchmarkHistoryBlobSink []byte
13		-	benchmarkHistoryBlobsSink [][]byte
	11	+	benchmarkHistoryViewSink HistoryView
	12	+	benchmarkHistorySnapshotSink HistorySnapshotView
	13	+	benchmarkHistoryBlobSink []byte
	14	+	benchmarkHistoryBlobsSink [][]byte
14	15		)
15	16
16	17		// TestHistoryViewPartialTailAllocationBounded guards the capture property used

@@ -75,6 +76,25 @@ func BenchmarkHistoryView10KRowsPartialTail(b *testing.B) {
75	76		b.ReportMetric(float64(10_000%256), "tailrows/view")
76	77		}
77	78
	79	+	func BenchmarkHistorySnapshotView10KRowsPartialTail(b *testing.B) {
	80	+	history := benchmarkHistory(b, 10_000, 120, 256)
	81	+	if got, want := len(history.tail), 16; got != want {
	82	+	b.Fatalf("partial tail rows = %d, want %d", got, want)
	83	+	}
	84	+	sealed := history.View().Chunk(0)
	85	+
	86	+	b.ReportAllocs()
	87	+	b.ResetTimer()
	88	+	for b.Loop() {
	89	+	benchmarkHistorySnapshotSink = history.SnapshotView()
	90	+	}
	91	+	b.StopTimer()
	92	+	if got := benchmarkHistorySnapshotSink; got.Chunk(0) != sealed \|\| got.Tail().Len() != 16 {
	93	+	b.Fatal("snapshot did not preserve chunk identity and tail rows")
	94	+	}
	95	+	b.ReportMetric(float64(benchmarkHistorySnapshotSink.ChunkCount()), "sealedchunks/view")
	96	+	}
	97	+
78	98		func BenchmarkMarshalHistoryChunk256x120(b *testing.B) {
79	99		history := benchmarkHistory(b, 256, 120, 256)
80	100		chunk := history.SealAndView().Chunk(0)


@@ -8,9 +8,10 @@ import (
8		)	8		)
9			9
10		var (	10		var (
11	-	benchmarkHistoryViewSink HistoryView	11	+	benchmarkHistoryViewSink HistoryView
12	-	benchmarkHistoryBlobSink []byte	12	+	benchmarkHistorySnapshotSink HistorySnapshotView
13	-	benchmarkHistoryBlobsSink [][]byte	13	+	benchmarkHistoryBlobSink []byte
			14	+	benchmarkHistoryBlobsSink [][]byte
14		)	15		)
15			16
16		// TestHistoryViewPartialTailAllocationBounded guards the capture property used	17		// TestHistoryViewPartialTailAllocationBounded guards the capture property used

@@ -75,6 +76,25 @@ func BenchmarkHistoryView10KRowsPartialTail(b *testing.B) {
75		b.ReportMetric(float64(10_000%256), "tailrows/view")	76		b.ReportMetric(float64(10_000%256), "tailrows/view")
76		}	77		}
77			78
			79	+	func BenchmarkHistorySnapshotView10KRowsPartialTail(b *testing.B) {
			80	+	history := benchmarkHistory(b, 10_000, 120, 256)
			81	+	if got, want := len(history.tail), 16; got != want {
			82	+	b.Fatalf("partial tail rows = %d, want %d", got, want)
			83	+	}
			84	+	sealed := history.View().Chunk(0)
			85	+
			86	+	b.ReportAllocs()
			87	+	b.ResetTimer()
			88	+	for b.Loop() {
			89	+	benchmarkHistorySnapshotSink = history.SnapshotView()
			90	+	}
			91	+	b.StopTimer()
			92	+	if got := benchmarkHistorySnapshotSink; got.Chunk(0) != sealed \|\| got.Tail().Len() != 16 {
			93	+	b.Fatal("snapshot did not preserve chunk identity and tail rows")
			94	+	}
			95	+	b.ReportMetric(float64(benchmarkHistorySnapshotSink.ChunkCount()), "sealedchunks/view")
			96	+	}
			97	+
78		func BenchmarkMarshalHistoryChunk256x120(b *testing.B) {	98		func BenchmarkMarshalHistoryChunk256x120(b *testing.B) {
79		history := benchmarkHistory(b, 256, 120, 256)	99		history := benchmarkHistory(b, 256, 120, 256)
80		chunk := history.SealAndView().Chunk(0)	100		chunk := history.SealAndView().Chunk(0)

M pkg/vt/history.go +15 -2


@@ -88,13 +88,15 @@ func (h *History) Append(row []renderer.Cell) error {
88	88		if len(row) > h.maxCells {
89	89		return ErrHistoryRowTooWide
90	90		}
	91	+	// Make room before adding so Cells cannot overflow even when the default
	92	+	// capacity is MaxInt.
	93	+	h.evictFor(len(row))
91	94		h.tail = append(h.tail, append([]renderer.Cell(nil), row...))
92	95		h.rows++
93	96		h.cells += len(row)
94	97		if len(h.tail) == h.chunkRows {
95	98		h.sealTail()
96	99		}
97		-	h.evict()
98	100		return nil
99	101		}
100	102

@@ -107,7 +109,18 @@ func (h *History) sealTail() {
107	109		}
108	110
109	111		func (h *History) evict() {
110		-	for h.rows > h.maxRows \|\| h.cells > h.maxCells {
	112	+	h.evictUntil(0, 0)
	113	+	}
	114	+
	115	+	// evictFor discards oldest rows until another row using cellCount cells can fit.
	116	+	func (h *History) evictFor(cellCount int) {
	117	+	h.evictUntil(1, cellCount)
	118	+	}
	119	+
	120	+	// evictUntil makes room for rowCount rows and cellCount cells without
	121	+	// overflowing either accounting total.
	122	+	func (h *History) evictUntil(rowCount, cellCount int) {
	123	+	for h.rows > h.maxRows-rowCount \|\| h.cells > h.maxCells-cellCount {
111	124		if len(h.chunks) > 0 {
112	125		chunk := h.chunks[0]
113	126		row := chunk.rows[0]

 	if len(row) > h.maxCells {
 		return ErrHistoryRowTooWide
 	}
+	// Make room before adding so Cells cannot overflow even when the default
+	// capacity is MaxInt.
+	h.evictFor(len(row))
 	h.tail = append(h.tail, append([]renderer.Cell(nil), row...))
 	h.rows++
 	h.cells += len(row)
 	if len(h.tail) == h.chunkRows {
 		h.sealTail()
 	}
-	h.evict()
 	return nil
 }
 }
 func (h *History) evict() {
-	for h.rows > h.maxRows || h.cells > h.maxCells {
+	h.evictUntil(0, 0)
+}
+// evictFor discards oldest rows until another row using cellCount cells can fit.
+func (h *History) evictFor(cellCount int) {
+	h.evictUntil(1, cellCount)
+}
+// evictUntil makes room for rowCount rows and cellCount cells without
+// overflowing either accounting total.
+func (h *History) evictUntil(rowCount, cellCount int) {
+	for h.rows > h.maxRows-rowCount || h.cells > h.maxCells-cellCount {
 		if len(h.chunks) > 0 {
 			chunk := h.chunks[0]
 			row := chunk.rows[0]

M pkg/vt/history_codec.go +4 -4


@@ -46,7 +46,7 @@ func MarshalHistory(view HistoryView) ([]byte, error) {
46	46		for _, row := range chunk.rows {
47	47		cellCount := uint64(len(row))
48	48		cellBytes, ok := historyCellByteCount(cellCount)
49		-	if cellCount > maxHistoryRowCells \|\| !ok \|\|
	49	+	if !ok \|\|
50	50		!addHistoryDecodeBudget(&stats.cells, cellCount, maxHistoryCells) \|\|
51	51		!addHistoryDecodeBudget(&stats.styles, cellCount, maxHistoryDecodeStyles) \|\|
52	52		!addHistoryDecodeBudget(&stats.bytes, cellBytes, maxHistoryDecodedBytes) {

@@ -193,7 +193,7 @@ func parseHistory(data []byte, populate bool) (HistoryView, historyDecodeStats,
193	193		for range rowCount {
194	194		cellCount, ok := p.uint32()
195	195		cellBytes, validCellBytes := historyCellByteCount(uint64(cellCount))
196		-	if !ok \|\| uint64(cellCount) > maxHistoryRowCells \|\| !validCellBytes \|\|
	196	+	if !ok \|\| !validCellBytes \|\|
197	197		uint64(cellCount) > uint64(len(p.data))/historyCellBytes \|\|
198	198		!addHistoryDecodeBudget(&stats.cells, uint64(cellCount), maxHistoryCells) \|\|
199	199		!addHistoryDecodeBudget(&stats.styles, uint64(cellCount), maxHistoryDecodeStyles) \|\|

@@ -205,7 +205,7 @@ func parseHistory(data []byte, populate bool) (HistoryView, historyDecodeStats,
205	205		if populate {
206	206		row = make([]renderer.Cell, cellCount)
207	207		}
208		-	for i := uint32(0); i < cellCount; i++ {
	208	+	for i := range cellCount {
209	209		cell, ok := p.cell()
210	210		if !ok \|\| !validHistoryCell(cell) {
211	211		return HistoryView{}, historyDecodeStats{}, false

@@ -225,7 +225,7 @@ func parseHistory(data []byte, populate bool) (HistoryView, historyDecodeStats,
225	225		if len(p.data) != 0 {
226	226		return HistoryView{}, historyDecodeStats{}, false
227	227		}
228		-	return HistoryView{chunks: chunks, rows: int(stats.rows)}, stats, true
	228	+	return HistoryView{chunks: chunks, rows: int(stats.rows), cells: int(stats.cells)}, stats, true
229	229		}
230	230
231	231		func historyCellByteCount(cellCount uint64) (uint64, bool) {


@@ -46,7 +46,7 @@ func MarshalHistory(view HistoryView) ([]byte, error) {
46		for _, row := range chunk.rows {	46		for _, row := range chunk.rows {
47		cellCount := uint64(len(row))	47		cellCount := uint64(len(row))
48		cellBytes, ok := historyCellByteCount(cellCount)	48		cellBytes, ok := historyCellByteCount(cellCount)
49	-	if cellCount > maxHistoryRowCells \|\| !ok \|\|	49	+	if !ok \|\|
50		!addHistoryDecodeBudget(&stats.cells, cellCount, maxHistoryCells) \|\|	50		!addHistoryDecodeBudget(&stats.cells, cellCount, maxHistoryCells) \|\|
51		!addHistoryDecodeBudget(&stats.styles, cellCount, maxHistoryDecodeStyles) \|\|	51		!addHistoryDecodeBudget(&stats.styles, cellCount, maxHistoryDecodeStyles) \|\|
52		!addHistoryDecodeBudget(&stats.bytes, cellBytes, maxHistoryDecodedBytes) {	52		!addHistoryDecodeBudget(&stats.bytes, cellBytes, maxHistoryDecodedBytes) {

@@ -193,7 +193,7 @@ func parseHistory(data []byte, populate bool) (HistoryView, historyDecodeStats,
193		for range rowCount {	193		for range rowCount {
194		cellCount, ok := p.uint32()	194		cellCount, ok := p.uint32()
195		cellBytes, validCellBytes := historyCellByteCount(uint64(cellCount))	195		cellBytes, validCellBytes := historyCellByteCount(uint64(cellCount))
196	-	if !ok \|\| uint64(cellCount) > maxHistoryRowCells \|\| !validCellBytes \|\|	196	+	if !ok \|\| !validCellBytes \|\|
197		uint64(cellCount) > uint64(len(p.data))/historyCellBytes \|\|	197		uint64(cellCount) > uint64(len(p.data))/historyCellBytes \|\|
198		!addHistoryDecodeBudget(&stats.cells, uint64(cellCount), maxHistoryCells) \|\|	198		!addHistoryDecodeBudget(&stats.cells, uint64(cellCount), maxHistoryCells) \|\|
199		!addHistoryDecodeBudget(&stats.styles, uint64(cellCount), maxHistoryDecodeStyles) \|\|	199		!addHistoryDecodeBudget(&stats.styles, uint64(cellCount), maxHistoryDecodeStyles) \|\|

@@ -205,7 +205,7 @@ func parseHistory(data []byte, populate bool) (HistoryView, historyDecodeStats,
205		if populate {	205		if populate {
206		row = make([]renderer.Cell, cellCount)	206		row = make([]renderer.Cell, cellCount)
207		}	207		}
208	-	for i := uint32(0); i < cellCount; i++ {	208	+	for i := range cellCount {
209		cell, ok := p.cell()	209		cell, ok := p.cell()
210		if !ok \|\| !validHistoryCell(cell) {	210		if !ok \|\| !validHistoryCell(cell) {
211		return HistoryView{}, historyDecodeStats{}, false	211		return HistoryView{}, historyDecodeStats{}, false

@@ -225,7 +225,7 @@ func parseHistory(data []byte, populate bool) (HistoryView, historyDecodeStats,
225		if len(p.data) != 0 {	225		if len(p.data) != 0 {
226		return HistoryView{}, historyDecodeStats{}, false	226		return HistoryView{}, historyDecodeStats{}, false
227		}	227		}
228	-	return HistoryView{chunks: chunks, rows: int(stats.rows)}, stats, true	228	+	return HistoryView{chunks: chunks, rows: int(stats.rows), cells: int(stats.cells)}, stats, true
229		}	229		}
230			230
231		func historyCellByteCount(cellCount uint64) (uint64, bool) {	231		func historyCellByteCount(cellCount uint64) (uint64, bool) {

M pkg/vt/history_codec_test.go +28 -7


@@ -2,6 +2,8 @@ package vt
2	2
3	3		import (
4	4		"encoding/binary"
	5	+	"fmt"
	6	+	"math"
5	7		"testing"
6	8
7	9		"github.com/bnema/vev/pkg/renderer"

@@ -123,10 +125,7 @@ func TestChunkCodecSupportsRepresentativeHistory(t *testing.T) {
123	125		}
124	126
125	127		func TestChunkCodecRejectsDimensionsBeyondSupportedLimits(t *testing.T) {
126		-	const (
127		-	supportedRows = 12_000
128		-	supportedWidth = 160
129		-	)
	128	+	const supportedRows = 12_000
130	129		tests := []struct {
131	130		name string
132	131		view HistoryView

@@ -138,9 +137,9 @@ func TestChunkCodecRejectsDimensionsBeyondSupportedLimits(t *testing.T) {
138	137		data: historyPayloadWithDimensions(supportedRows+1, 0),
139	138		},
140	139		{
141		-	name: "too many cells in a row",
142		-	view: historyViewWithDimensions(1, supportedWidth+1),
143		-	data: historyPayloadWithDimensions(1, supportedWidth+1),
	140	+	name: "too many aggregate cells",
	141	+	view: historyViewWithDimensions(maxHistoryCells+1, 0),
	142	+	data: historyPayloadWithDimensions(maxHistoryRows+1, 0),
144	143		},
145	144		}
146	145

@@ -156,6 +155,28 @@ func TestChunkCodecRejectsDimensionsBeyondSupportedLimits(t *testing.T) {
156	155		}
157	156		}
158	157
	158	+	func TestChunkCodecAcceptsWideRowsWithinAggregateBudget(t *testing.T) {
	159	+	for _, width := range []int{161, 293, math.MaxUint16} {
	160	+	t.Run(fmt.Sprintf("width_%d", width), func(t *testing.T) {
	161	+	view := historyViewWithDimensions(1, width)
	162	+	encoded, err := MarshalHistory(view)
	163	+	if err != nil {
	164	+	t.Fatalf("marshal width %d: %v", width, err)
	165	+	}
	166	+	decoded, err := UnmarshalHistory(encoded)
	167	+	if err != nil {
	168	+	t.Fatalf("unmarshal width %d: %v", width, err)
	169	+	}
	170	+	if got := len(decoded.Row(0)); got != width {
	171	+	t.Fatalf("decoded width = %d, want %d", got, width)
	172	+	}
	173	+	if stats, err := PreflightHistoryBlob(encoded); err != nil \|\| stats.Cells != uint64(width) {
	174	+	t.Fatalf("preflight stats = %+v, err = %v", stats, err)
	175	+	}
	176	+	})
	177	+	}
	178	+	}
	179	+
159	180		func TestChunkCodecRejectsHostileChunkAndRowDeclarations(t *testing.T) {
160	181		tests := []struct {
161	182		name string

 import (
 	"encoding/binary"
+	"fmt"
+	"math"
 	"testing"
 	"github.com/bnema/vev/pkg/renderer"
 }
 func TestChunkCodecRejectsDimensionsBeyondSupportedLimits(t *testing.T) {
-	const (
+	const supportedRows = 12_000
-		supportedRows  = 12_000
-		supportedWidth = 160
-	)
 	tests := []struct {
 		name string
 		view HistoryView
 			data: historyPayloadWithDimensions(supportedRows+1, 0),
 		},
 		{
-			name: "too many cells in a row",
+			name: "too many aggregate cells",
-			view: historyViewWithDimensions(1, supportedWidth+1),
+			view: historyViewWithDimensions(maxHistoryCells+1, 0),
-			data: historyPayloadWithDimensions(1, supportedWidth+1),
+			data: historyPayloadWithDimensions(maxHistoryRows+1, 0),
 		},
 	}
 	}
 }
+func TestChunkCodecAcceptsWideRowsWithinAggregateBudget(t *testing.T) {
+	for _, width := range []int{161, 293, math.MaxUint16} {
+		t.Run(fmt.Sprintf("width_%d", width), func(t *testing.T) {
+			view := historyViewWithDimensions(1, width)
+			encoded, err := MarshalHistory(view)
+			if err != nil {
+				t.Fatalf("marshal width %d: %v", width, err)
+			}
+			decoded, err := UnmarshalHistory(encoded)
+			if err != nil {
+				t.Fatalf("unmarshal width %d: %v", width, err)
+			}
+			if got := len(decoded.Row(0)); got != width {
+				t.Fatalf("decoded width = %d, want %d", got, width)
+			}
+			if stats, err := PreflightHistoryBlob(encoded); err != nil || stats.Cells != uint64(width) {
+				t.Fatalf("preflight stats = %+v, err = %v", stats, err)
+			}
+		})
+	}
+}
 func TestChunkCodecRejectsHostileChunkAndRowDeclarations(t *testing.T) {
 	tests := []struct {
 		name       string

M pkg/vt/snapshot.go +2 -0


@@ -90,6 +90,7 @@ func HistoryFromBlobs(config HistoryConfig, sealed [][]byte, tail []byte) (*Hist
90	90		if err != nil \|\| len(view.chunks) != 1 \|\| len(view.chunks[0].rows) == 0 {
91	91		return nil, fmt.Errorf("restore sealed history: %w", errInvalidHistory)
92	92		}
	93	+	h.evictUntil(len(view.chunks[0].rows), view.Cells())
93	94		h.chunks = append(h.chunks, view.chunks[0])
94	95		h.rows += len(view.chunks[0].rows)
95	96		h.cells += view.Cells()

@@ -99,6 +100,7 @@ func HistoryFromBlobs(config HistoryConfig, sealed [][]byte, tail []byte) (*Hist
99	100		return nil, fmt.Errorf("restore history tail: %w", errInvalidHistory)
100	101		}
101	102		if len(view.chunks) == 1 {
	103	+	h.evictUntil(len(view.chunks[0].rows), view.Cells())
102	104		h.tail = view.chunks[0].rows
103	105		h.rows += len(h.tail)
104	106		h.cells += view.Cells()


@@ -90,6 +90,7 @@ func HistoryFromBlobs(config HistoryConfig, sealed [][]byte, tail []byte) (*Hist
90	if err != nil \|\| len(view.chunks) != 1 \|\| len(view.chunks[0].rows) == 0 {	90		if err != nil \|\| len(view.chunks) != 1 \|\| len(view.chunks[0].rows) == 0 {
91	return nil, fmt.Errorf("restore sealed history: %w", errInvalidHistory)	91		return nil, fmt.Errorf("restore sealed history: %w", errInvalidHistory)
92	}	92		}
		93	+	h.evictUntil(len(view.chunks[0].rows), view.Cells())
93	h.chunks = append(h.chunks, view.chunks[0])	94		h.chunks = append(h.chunks, view.chunks[0])
94	h.rows += len(view.chunks[0].rows)	95		h.rows += len(view.chunks[0].rows)
95	h.cells += view.Cells()	96		h.cells += view.Cells()

@@ -99,6 +100,7 @@ func HistoryFromBlobs(config HistoryConfig, sealed [][]byte, tail []byte) (*Hist
99	return nil, fmt.Errorf("restore history tail: %w", errInvalidHistory)	100		return nil, fmt.Errorf("restore history tail: %w", errInvalidHistory)
100	}	101		}
101	if len(view.chunks) == 1 {	102		if len(view.chunks) == 1 {
		103	+	h.evictUntil(len(view.chunks[0].rows), view.Cells())
102	h.tail = view.chunks[0].rows	104		h.tail = view.chunks[0].rows
103	h.rows += len(h.tail)	105		h.rows += len(h.tail)
104	h.cells += view.Cells()	106		h.cells += view.Cells()