merge(perf): history tail reuse, style maps and row ID sets

commit: 73481a8071e48d5942b3f7bd8c7accb03c14afbe
author: brice · 20d ago · 2026-10-02 09:46 UTC
merge of: b58d93f + 9dd2424 · diff against the first parent
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

5 files · +480 -29

M history.go +86 -14


@@ -39,6 +39,7 @@ type History struct {
39	39		styleScratch map[renderer.Style]struct{}
40	40		payloadScratch map[renderer.CellPayload]struct{}
41	41		tailPagePayloads map[renderer.CellPayload]struct{}
	42	+	styleRowScratch map[renderer.Style]int
42	43		rows int
43	44		cells int
44	45		logicalBytes uint64

@@ -62,6 +63,28 @@ type HistoryChunk struct {
62	63		}
63	64
64	65		func newHistoryChunks(rows [][]renderer.Cell, bounds []LineBound, rowIDs []RowID) []*HistoryChunk {
	66	+	return newHistoryChunksScratch(rows, bounds, rowIDs, nil)
	67	+	}
	68	+
	69	+	// maxRetainedStyleScratch bounds the style-set capacity kept between uses so
	70	+	// one pathological high-cardinality page cannot pin a large map forever.
	71	+	const maxRetainedStyleScratch = 1024
	72	+
	73	+	// newHistoryChunksScratch is newHistoryChunks with an optional caller-owned
	74	+	// map (scratch may be nil) reused, cleared, across chunks instead of allocating
	75	+	// one per chunk. The map is replaced before any reuse if the previous chunk
	76	+	// grew it past maxRetainedStyleScratch, and dropped on return if the last one
	77	+	// did, so one high-cardinality chunk cannot pin a large map.
	78	+	func newHistoryChunksScratch(rows [][]renderer.Cell, bounds []LineBound, rowIDs []RowID, scratch map[renderer.Style]int) []HistoryChunk {
	79	+	var local map[renderer.Style]int
	80	+	if scratch == nil {
	81	+	scratch = &local
	82	+	}
	83	+	defer func() {
	84	+	if len(*scratch) > maxRetainedStyleScratch {
	85	+	*scratch = nil
	86	+	}
	87	+	}()
65	88		bounds = growBounds(bounds, len(rows))
66	89		rowIDs = growRowIDs(rowIDs, len(rows))
67	90		var chunks []*HistoryChunk

@@ -73,12 +96,26 @@ func newHistoryChunks(rows [][]renderer.Cell, bounds []LineBound, rowIDs []RowID
73	96		end++
74	97		}
75	98		frame := renderer.NewFrame(width, end-start)
76		-	lastStyleRow := make(map[renderer.Style]int)
	99	+	if scratch == nil \|\| len(scratch) > maxRetainedStyleScratch {
	100	+	*scratch = make(map[renderer.Style]int)
	101	+	} else {
	102	+	clear(*scratch)
	103	+	}
	104	+	lastStyleRow := *scratch
	105	+	defaultStyle := renderer.DefaultStyle()
77	106		for i := start; i < end; i++ {
78	107		frame.WriteRow(i-start, 0, rows[i])
79		-	for _, cell := range rows[i] {
	108	+	// A run of identical raw styles canonicalizes identically and is
	109	+	// already recorded for this row, so only the first cell of a run
	110	+	// pays for Canonical and the map write.
	111	+	var previous renderer.Style
	112	+	for j, cell := range rows[i] {
	113	+	if j > 0 && cell.Style == previous {
	114	+	continue
	115	+	}
	116	+	previous = cell.Style
80	117		style := cell.Style.Canonical()
81		-	if style != renderer.DefaultStyle() {
	118	+	if style != defaultStyle {
82	119		lastStyleRow[style] = i - start
83	120		}
84	121		}

@@ -300,8 +337,8 @@ func (h *History) recordTailRowStyles(width int) {
300	337		if h.tailPageRows == 0 \|\| h.tailPageWidth != width \|\| h.tailPageRows >= maxHistorySlabRows(width) {
301	338		h.tailPageWidth = width
302	339		h.tailPageRows = 0
303		-	h.tailPageStyles = map[renderer.Style]struct{}{renderer.DefaultStyle(): {}}
304		-	h.tailPagePayloads = nil
	340	+	h.resetTailPageSets()
	341	+	h.tailPageStyles[renderer.DefaultStyle()] = struct{}{}
305	342		}
306	343		for p := range h.payloadScratch {
307	344		if h.tailPagePayloads == nil {

@@ -315,13 +352,28 @@ func (h *History) recordTailRowStyles(width int) {
315	352		h.tailPageRows++
316	353		}
317	354
	355	+	// resetTailPageSets empties the per-page style and payload sets, keeping their
	356	+	// storage for the next page unless a pathological page grew them past
	357	+	// maxRetainedStyleScratch. The style set is always non-nil afterwards.
	358	+	func (h *History) resetTailPageSets() {
	359	+	if h.tailPageStyles == nil \|\| len(h.tailPageStyles) > maxRetainedStyleScratch {
	360	+	h.tailPageStyles = make(map[renderer.Style]struct{})
	361	+	} else {
	362	+	clear(h.tailPageStyles)
	363	+	}
	364	+	if len(h.tailPagePayloads) > maxRetainedStyleScratch {
	365	+	h.tailPagePayloads = nil
	366	+	} else {
	367	+	clear(h.tailPagePayloads)
	368	+	}
	369	+	}
	370	+
318	371		func (h *History) rebuildTailAccounting() {
319	372		old := h.tailBytes
320	373		h.tailBytes = 0
321	374		h.tailPageWidth = 0
322	375		h.tailPageRows = 0
323		-	h.tailPageStyles = nil
324		-	h.tailPagePayloads = nil
	376	+	h.resetTailPageSets()
325	377		for _, row := range h.tail {
326	378		h.prepareRowStyles(row)
327	379		delta := h.tailAppendDelta(len(row))

@@ -496,16 +548,33 @@ func (h *History) sealTail() {
496	548		chunks := h.tailViewChunks()
497	549		h.chunks = append(h.chunks, chunks...)
498	550		h.logicalBytes = h.logicalBytes - h.tailBytes + historyChunksLogicalBytes(chunks)
499		-	h.tail = nil
500	551		h.cachedTail = nil
501		-	h.tailCells = nil
502		-	h.tailBounds = nil
503		-	h.tailIDs = nil
	552	+	// The sealed chunks hold compact copies (newHistoryChunks and the bounds and
	553	+	// ID slices are copied), and no view, snapshot or callback ever receives
	554	+	// h.tail rows or h.tailCells. The mutable-tail storage is therefore private
	555	+	// and can be recycled for the next chunk instead of reallocated. Cells are
	556	+	// cleared so recycled storage does not pin payload strings. Oversized
	557	+	// storage (wide rows) is released as before.
	558	+	clear(h.tail)
	559	+	h.tail = h.tail[:0]
	560	+	if cap(h.tailCells) > maxTailPreallocCells {
	561	+	h.tailCells = nil
	562	+	} else {
	563	+	clear(h.tailCells)
	564	+	h.tailCells = h.tailCells[:0]
	565	+	}
	566	+	h.tailBounds = h.tailBounds[:0]
	567	+	h.tailIDs = h.tailIDs[:0]
504	568		h.tailBytes = 0
505	569		h.tailPageWidth = 0
506	570		h.tailPageRows = 0
507		-	h.tailPageStyles = nil
508		-	h.tailPagePayloads = nil
	571	+	// Sets are cleared lazily by the next page; release oversized ones now.
	572	+	if len(h.tailPageStyles) > maxRetainedStyleScratch {
	573	+	h.tailPageStyles = nil
	574	+	}
	575	+	if len(h.tailPagePayloads) > maxRetainedStyleScratch {
	576	+	h.tailPagePayloads = nil
	577	+	}
509	578		}
510	579
511	580		// evictFor discards oldest rows until row can fit both retention budgets.

@@ -552,6 +621,9 @@ func (h *History) evictOldest() {
552	621		h.cells -= len(row)
553	622		h.tail[0] = nil
554	623		h.tail = h.tail[1:]
	624	+	// The evicted prefix stays in the recycled backing array; clear it so its
	625	+	// payload strings are not pinned until the next seal.
	626	+	clear(row)
555	627		h.tailCells = h.tailCells[len(row):]
556	628		h.tailBounds = growBounds(h.tailBounds, len(h.tail)+1)[1:]
557	629		h.tailIDs = growRowIDs(h.tailIDs, len(h.tail)+1)[1:]

@@ -569,7 +641,7 @@ func (h *History) SealAndView() HistoryView {
569	641
570	642		func (h History) tailViewChunks() []HistoryChunk {
571	643		if h.cachedTail == nil && len(h.tail) > 0 {
572		-	h.cachedTail = newHistoryChunks(h.tail, h.tailBounds, h.tailIDs)
	644	+	h.cachedTail = newHistoryChunksScratch(h.tail, h.tailBounds, h.tailIDs, &h.styleRowScratch)
573	645		}
574	646		return h.cachedTail
575	647		}

 	styleScratch      map[renderer.Style]struct{}
 	payloadScratch    map[renderer.CellPayload]struct{}
 	tailPagePayloads  map[renderer.CellPayload]struct{}
+	styleRowScratch   map[renderer.Style]int
 	rows              int
 	cells             int
 	logicalBytes      uint64
 }
 func newHistoryChunks(rows [][]renderer.Cell, bounds []LineBound, rowIDs []RowID) []*HistoryChunk {
+	return newHistoryChunksScratch(rows, bounds, rowIDs, nil)
+}
+// maxRetainedStyleScratch bounds the style-set capacity kept between uses so
+// one pathological high-cardinality page cannot pin a large map forever.
+const maxRetainedStyleScratch = 1024
+// newHistoryChunksScratch is newHistoryChunks with an optional caller-owned
+// map (scratch may be nil) reused, cleared, across chunks instead of allocating
+// one per chunk. The map is replaced before any reuse if the previous chunk
+// grew it past maxRetainedStyleScratch, and dropped on return if the last one
+// did, so one high-cardinality chunk cannot pin a large map.
+func newHistoryChunksScratch(rows [][]renderer.Cell, bounds []LineBound, rowIDs []RowID, scratch *map[renderer.Style]int) []*HistoryChunk {
+	var local map[renderer.Style]int
+	if scratch == nil {
+		scratch = &local
+	}
+	defer func() {
+		if len(*scratch) > maxRetainedStyleScratch {
+			*scratch = nil
+		}
+	}()
 	bounds = growBounds(bounds, len(rows))
 	rowIDs = growRowIDs(rowIDs, len(rows))
 	var chunks []*HistoryChunk
 			end++
 		}
 		frame := renderer.NewFrame(width, end-start)
-		lastStyleRow := make(map[renderer.Style]int)
+		if *scratch == nil || len(*scratch) > maxRetainedStyleScratch {
+			*scratch = make(map[renderer.Style]int)
+		} else {
+			clear(*scratch)
+		}
+		lastStyleRow := *scratch
+		defaultStyle := renderer.DefaultStyle()
 		for i := start; i < end; i++ {
 			frame.WriteRow(i-start, 0, rows[i])
-			for _, cell := range rows[i] {
+			// A run of identical raw styles canonicalizes identically and is
+			// already recorded for this row, so only the first cell of a run
+			// pays for Canonical and the map write.
+			var previous renderer.Style
+			for j, cell := range rows[i] {
+				if j > 0 && cell.Style == previous {
+					continue
+				}
+				previous = cell.Style
 				style := cell.Style.Canonical()
-				if style != renderer.DefaultStyle() {
+				if style != defaultStyle {
 					lastStyleRow[style] = i - start
 				}
 			}
 	if h.tailPageRows == 0 || h.tailPageWidth != width || h.tailPageRows >= maxHistorySlabRows(width) {
 		h.tailPageWidth = width
 		h.tailPageRows = 0
-		h.tailPageStyles = map[renderer.Style]struct{}{renderer.DefaultStyle(): {}}
+		h.resetTailPageSets()
-		h.tailPagePayloads = nil
+		h.tailPageStyles[renderer.DefaultStyle()] = struct{}{}
 	}
 	for p := range h.payloadScratch {
 		if h.tailPagePayloads == nil {
 	h.tailPageRows++
 }
+// resetTailPageSets empties the per-page style and payload sets, keeping their
+// storage for the next page unless a pathological page grew them past
+// maxRetainedStyleScratch. The style set is always non-nil afterwards.
+func (h *History) resetTailPageSets() {
+	if h.tailPageStyles == nil || len(h.tailPageStyles) > maxRetainedStyleScratch {
+		h.tailPageStyles = make(map[renderer.Style]struct{})
+	} else {
+		clear(h.tailPageStyles)
+	}
+	if len(h.tailPagePayloads) > maxRetainedStyleScratch {
+		h.tailPagePayloads = nil
+	} else {
+		clear(h.tailPagePayloads)
+	}
+}
 func (h *History) rebuildTailAccounting() {
 	old := h.tailBytes
 	h.tailBytes = 0
 	h.tailPageWidth = 0
 	h.tailPageRows = 0
-	h.tailPageStyles = nil
+	h.resetTailPageSets()
-	h.tailPagePayloads = nil
 	for _, row := range h.tail {
 		h.prepareRowStyles(row)
 		delta := h.tailAppendDelta(len(row))
 	chunks := h.tailViewChunks()
 	h.chunks = append(h.chunks, chunks...)
 	h.logicalBytes = h.logicalBytes - h.tailBytes + historyChunksLogicalBytes(chunks)
-	h.tail = nil
 	h.cachedTail = nil
-	h.tailCells = nil
+	// The sealed chunks hold compact copies (newHistoryChunks and the bounds and
-	h.tailBounds = nil
+	// ID slices are copied), and no view, snapshot or callback ever receives
-	h.tailIDs = nil
+	// h.tail rows or h.tailCells. The mutable-tail storage is therefore private
+	// and can be recycled for the next chunk instead of reallocated. Cells are
+	// cleared so recycled storage does not pin payload strings. Oversized
+	// storage (wide rows) is released as before.
+	clear(h.tail)
+	h.tail = h.tail[:0]
+	if cap(h.tailCells) > maxTailPreallocCells {
+		h.tailCells = nil
+	} else {
+		clear(h.tailCells)
+		h.tailCells = h.tailCells[:0]
+	}
+	h.tailBounds = h.tailBounds[:0]
+	h.tailIDs = h.tailIDs[:0]
 	h.tailBytes = 0
 	h.tailPageWidth = 0
 	h.tailPageRows = 0
-	h.tailPageStyles = nil
+	// Sets are cleared lazily by the next page; release oversized ones now.
-	h.tailPagePayloads = nil
+	if len(h.tailPageStyles) > maxRetainedStyleScratch {
+		h.tailPageStyles = nil
+	}
+	if len(h.tailPagePayloads) > maxRetainedStyleScratch {
+		h.tailPagePayloads = nil
+	}
 }
 // evictFor discards oldest rows until row can fit both retention budgets.
 	h.cells -= len(row)
 	h.tail[0] = nil
 	h.tail = h.tail[1:]
+	// The evicted prefix stays in the recycled backing array; clear it so its
+	// payload strings are not pinned until the next seal.
+	clear(row)
 	h.tailCells = h.tailCells[len(row):]
 	h.tailBounds = growBounds(h.tailBounds, len(h.tail)+1)[1:]
 	h.tailIDs = growRowIDs(h.tailIDs, len(h.tail)+1)[1:]
 func (h *History) tailViewChunks() []*HistoryChunk {
 	if h.cachedTail == nil && len(h.tail) > 0 {
-		h.cachedTail = newHistoryChunks(h.tail, h.tailBounds, h.tailIDs)
+		h.cachedTail = newHistoryChunksScratch(h.tail, h.tailBounds, h.tailIDs, &h.styleRowScratch)
 	}
 	return h.cachedTail
 }

M history_codec.go +7 -5


@@ -234,7 +234,7 @@ func parseHistory(data []byte, populate bool) (HistoryView, historyDecodeStats,
234	234		return invalid()
235	235		}
236	236		stats := historyDecodeStats{chunks: uint64(chunkCount)}
237		-	seenIDs := make(map[RowID]struct{})
	237	+	var seenIDs rowIDSet
238	238		var maxID RowID
239	239		var chunks []*HistoryChunk
240	240		if populate {

@@ -280,6 +280,7 @@ func parseHistory(data []byte, populate bool) (HistoryView, historyDecodeStats,
280	280		return invalid()
281	281		}
282	282		usedPayloads := make([]bool, npayloads)
	283	+	seenIDs.begin(int(rows))
283	284		rowIDs := make([]RowID, rows)
284	285		bounds := make([]LineBound, rows)
285	286		for row := range rows {

@@ -288,13 +289,14 @@ func parseHistory(data []byte, populate bool) (HistoryView, historyDecodeStats,
288	289		if !ok \|\| !bok \|\| id == 0 \|\| id >= math.MaxUint64-1 \|\| !validHistoryBound(bound, int(width)) {
289	290		return invalid()
290	291		}
291		-	if _, duplicate := seenIDs[RowID(id)]; duplicate {
292		-	return invalid()
293		-	}
294		-	seenIDs[RowID(id)] = struct{}{}
	292	+	seenIDs.collect(RowID(id))
295	293		maxID = max(maxID, RowID(id))
296	294		rowIDs[row], bounds[row] = RowID(id), bound
297	295		}
	296	+	// Reject duplicate IDs before decoding any cells, as the map did.
	297	+	if !seenIDs.commit() {
	298	+	return invalid()
	299	+	}
298	300		lastStyleRow := make([]int, nstyles)
299	301		for i := range lastStyleRow {
300	302		lastStyleRow[i] = -1

 		return invalid()
 	}
 	stats := historyDecodeStats{chunks: uint64(chunkCount)}
-	seenIDs := make(map[RowID]struct{})
+	var seenIDs rowIDSet
 	var maxID RowID
 	var chunks []*HistoryChunk
 	if populate {
 			return invalid()
 		}
 		usedPayloads := make([]bool, npayloads)
+		seenIDs.begin(int(rows))
 		rowIDs := make([]RowID, rows)
 		bounds := make([]LineBound, rows)
 		for row := range rows {
 			if !ok || !bok || id == 0 || id >= math.MaxUint64-1 || !validHistoryBound(bound, int(width)) {
 				return invalid()
 			}
-			if _, duplicate := seenIDs[RowID(id)]; duplicate {
+			seenIDs.collect(RowID(id))
-				return invalid()
-			}
-			seenIDs[RowID(id)] = struct{}{}
 			maxID = max(maxID, RowID(id))
 			rowIDs[row], bounds[row] = RowID(id), bound
 		}
+		// Reject duplicate IDs before decoding any cells, as the map did.
+		if !seenIDs.commit() {
+			return invalid()
+		}
 		lastStyleRow := make([]int, nstyles)
 		for i := range lastStyleRow {
 			lastStyleRow[i] = -1

A history_rowids.go +64 -0

@@ -0,0 +1,64 @@
	1	+	package vt
	2	+
	3	+	import "slices"
	4	+
	5	+	// rowIDSet detects duplicate row IDs across one or more decoded blobs without
	6	+	// a hash map in the common case. Row IDs are NOT required to be monotonic
	7	+	// (History.AppendWithID accepts any unused ID, so persisted blobs may list IDs
	8	+	// out of order), so this is an exact set, not an ordering check: it accepts
	9	+	// and rejects precisely the inputs a map[RowID]struct{} would.
	10	+	//
	11	+	// While every view contributes IDs strictly above everything seen so far, IDs
	12	+	// are kept in an ascending slice. The first view that interleaves with earlier
	13	+	// IDs converts the set to a map; from then on it behaves like a plain set.
	14	+	type rowIDSet struct {
	15	+	sorted []RowID
	16	+	members map[RowID]struct{}
	17	+	scratch []RowID
	18	+	}
	19	+
	20	+	// begin discards any IDs collected but not committed and reserves room for n.
	21	+	func (s *rowIDSet) begin(n int) { s.scratch = slices.Grow(s.scratch[:0], n) }
	22	+
	23	+	// reserve makes room for n more collected IDs.
	24	+	func (s *rowIDSet) reserve(n int) { s.scratch = slices.Grow(s.scratch, n) }
	25	+
	26	+	// collect records one ID of the view being collected.
	27	+	func (s *rowIDSet) collect(id RowID) { s.scratch = append(s.scratch, id) }
	28	+
	29	+	// commit reports whether the collected IDs are distinct from each other and
	30	+	// from every previously committed ID, and if so adds them to the set.
	31	+	func (s *rowIDSet) commit() bool {
	32	+	ids := s.scratch
	33	+	s.scratch = s.scratch[:0]
	34	+	if len(ids) == 0 {
	35	+	return true
	36	+	}
	37	+	slices.Sort(ids) // the scratch copy is private, so sorting in place is safe
	38	+
	39	+	for i := 1; i < len(ids); i++ {
	40	+	if ids[i] == ids[i-1] {
	41	+	return false
	42	+	}
	43	+	}
	44	+	if s.members == nil {
	45	+	if len(s.sorted) == 0 \|\| ids[0] > s.sorted[len(s.sorted)-1] {
	46	+	s.sorted = append(s.sorted, ids...)
	47	+	return true
	48	+	}
	49	+	s.members = make(map[RowID]struct{}, len(s.sorted)+len(ids))
	50	+	for _, id := range s.sorted {
	51	+	s.members[id] = struct{}{}
	52	+	}
	53	+	s.sorted = nil
	54	+	}
	55	+	for _, id := range ids {
	56	+	if _, duplicate := s.members[id]; duplicate {
	57	+	return false
	58	+	}
	59	+	}
	60	+	for _, id := range ids {
	61	+	s.members[id] = struct{}{}
	62	+	}
	63	+	return true
	64	+	}

@@ -0,0 +1,64 @@
	1	+	package vt
	2	+
	3	+	import "slices"
	4	+
	5	+	// rowIDSet detects duplicate row IDs across one or more decoded blobs without
	6	+	// a hash map in the common case. Row IDs are NOT required to be monotonic
	7	+	// (History.AppendWithID accepts any unused ID, so persisted blobs may list IDs
	8	+	// out of order), so this is an exact set, not an ordering check: it accepts
	9	+	// and rejects precisely the inputs a map[RowID]struct{} would.
	10	+	//
	11	+	// While every view contributes IDs strictly above everything seen so far, IDs
	12	+	// are kept in an ascending slice. The first view that interleaves with earlier
	13	+	// IDs converts the set to a map; from then on it behaves like a plain set.
	14	+	type rowIDSet struct {
	15	+	sorted []RowID
	16	+	members map[RowID]struct{}
	17	+	scratch []RowID
	18	+	}
	19	+
	20	+	// begin discards any IDs collected but not committed and reserves room for n.
	21	+	func (s *rowIDSet) begin(n int) { s.scratch = slices.Grow(s.scratch[:0], n) }
	22	+
	23	+	// reserve makes room for n more collected IDs.
	24	+	func (s *rowIDSet) reserve(n int) { s.scratch = slices.Grow(s.scratch, n) }
	25	+
	26	+	// collect records one ID of the view being collected.
	27	+	func (s *rowIDSet) collect(id RowID) { s.scratch = append(s.scratch, id) }
	28	+
	29	+	// commit reports whether the collected IDs are distinct from each other and
	30	+	// from every previously committed ID, and if so adds them to the set.
	31	+	func (s *rowIDSet) commit() bool {
	32	+	ids := s.scratch
	33	+	s.scratch = s.scratch[:0]
	34	+	if len(ids) == 0 {
	35	+	return true
	36	+	}
	37	+	slices.Sort(ids) // the scratch copy is private, so sorting in place is safe
	38	+
	39	+	for i := 1; i < len(ids); i++ {
	40	+	if ids[i] == ids[i-1] {
	41	+	return false
	42	+	}
	43	+	}
	44	+	if s.members == nil {
	45	+	if len(s.sorted) == 0 \|\| ids[0] > s.sorted[len(s.sorted)-1] {
	46	+	s.sorted = append(s.sorted, ids...)
	47	+	return true
	48	+	}
	49	+	s.members = make(map[RowID]struct{}, len(s.sorted)+len(ids))
	50	+	for _, id := range s.sorted {
	51	+	s.members[id] = struct{}{}
	52	+	}
	53	+	s.sorted = nil
	54	+	}
	55	+	for _, id := range ids {
	56	+	if _, duplicate := s.members[id]; duplicate {
	57	+	return false
	58	+	}
	59	+	}
	60	+	for _, id := range ids {
	61	+	s.members[id] = struct{}{}
	62	+	}
	63	+	return true
	64	+	}

A history_tail_reuse_test.go +314 -0

@@ -0,0 +1,314 @@
	1	+	package vt
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"math/rand"
	6	+	"testing"
	7	+
	8	+	renderer "github.com/bnema/vev-vt/core"
	9	+	"github.com/stretchr/testify/require"
	10	+	)
	11	+
	12	+	func styledRow(text string, fg int) []renderer.Cell {
	13	+	row := historyRow(text)
	14	+	for i := range row {
	15	+	row[i].Style = renderer.Style{Bold: true, Foreground: fg}
	16	+	}
	17	+	return row
	18	+	}
	19	+
	20	+	func rangeTexts(t *testing.T, v HistoryView) []string {
	21	+	t.Helper()
	22	+	var out []string
	23	+	require.NoError(t, v.Range(func(row []renderer.Cell) bool {
	24	+	out = append(out, rowText(row))
	25	+	return true
	26	+	}))
	27	+	return out
	28	+	}
	29	+
	30	+	// Sealing recycles the private tail storage. No previously handed-out view,
	31	+	// snapshot tail or Range row may observe later appends or seals.
	32	+	func TestHistoryTailStorageReuseKeepsViewsStable(t *testing.T) {
	33	+	h := NewHistory(HistoryConfig{MaxRows: 64, MaxBytes: 1 << 20, ChunkRows: 4})
	34	+	appendRow := func(i int) {
	35	+	require.NoError(t, h.Append(styledRow(fmt.Sprintf("r%03d", i), i%5+1), LineBound{End: 4}))
	36	+	}
	37	+	for i := range 2 {
	38	+	appendRow(i)
	39	+	}
	40	+	partial := h.View()
	41	+	snap := h.SnapshotView()
	42	+	var ranged [][]renderer.Cell
	43	+	require.NoError(t, partial.Range(func(row []renderer.Cell) bool { ranged = append(ranged, row); return true }))
	44	+
	45	+	for i := 2; i < 14; i++ { // seals three times, reusing storage each time
	46	+	appendRow(i)
	47	+	}
	48	+	sealedView := h.SealAndView()
	49	+	appendRow(14)
	50	+	appendRow(15)
	51	+
	52	+	require.Equal(t, []string{"r000", "r001"}, rangeTexts(t, partial))
	53	+	require.Equal(t, []string{"r000", "r001"}, rangeTexts(t, snap.Tail()))
	54	+	require.Equal(t, "r000", rowText(ranged[0]))
	55	+	require.Equal(t, "r001", rowText(ranged[1]))
	56	+	require.Equal(t, 14, sealedView.Len())
	57	+	texts := rangeTexts(t, sealedView)
	58	+	for i, got := range texts {
	59	+	require.Equal(t, fmt.Sprintf("r%03d", i), got)
	60	+	}
	61	+	live := rangeTexts(t, h.View())
	62	+	require.Len(t, live, 16)
	63	+	require.Equal(t, "r015", live[15])
	64	+	for _, c := range h.chunks {
	65	+	require.NoError(t, c.CheckInvariants())
	66	+	}
	67	+	}
	68	+
	69	+	func TestHistoryTailStorageIsRecycledAcrossSeals(t *testing.T) {
	70	+	h := NewHistory(HistoryConfig{MaxRows: 64, MaxBytes: 1 << 20, ChunkRows: 4})
	71	+	for i := range 4 {
	72	+	require.NoError(t, h.Append(historyRow(fmt.Sprintf("r%03d", i)), LineBound{End: 4}))
	73	+	}
	74	+	require.Empty(t, h.tail)
	75	+	require.Zero(t, len(h.tailCells))
	76	+	require.NotZero(t, cap(h.tailCells), "seal must keep tail storage for the next chunk")
	77	+	first := &h.tailCells[:1][0]
	78	+	for i := 4; i < 8; i++ {
	79	+	require.NoError(t, h.Append(historyRow(fmt.Sprintf("r%03d", i)), LineBound{End: 4}))
	80	+	}
	81	+	require.Same(t, first, &h.tailCells[:1][0], "storage reallocated across seals")
	82	+	// A reused backing array must not pin payload strings of sealed rows.
	83	+	payload, err := renderer.NewCellPayload("é\u0301", "https://example.test")
	84	+	require.NoError(t, err)
	85	+	row := historyRow("abcd")
	86	+	row[0].Payload = payload
	87	+	for range 4 {
	88	+	require.NoError(t, h.Append(row, LineBound{End: 4}))
	89	+	}
	90	+	for _, c := range h.tailCells[:cap(h.tailCells)] {
	91	+	require.True(t, c.Payload.Empty())
	92	+	}
	93	+	require.NoError(t, h.chunks[len(h.chunks)-1].CheckInvariants())
	94	+	}
	95	+
	96	+	// Eviction from a partial tail must not leave the evicted rows' payload
	97	+	// strings pinned in the recycled backing array's prefix.
	98	+	func TestHistoryTailEvictionClearsEvictedPrefix(t *testing.T) {
	99	+	payload, err := renderer.NewCellPayload("é\u0301", "https://example.test")
	100	+	require.NoError(t, err)
	101	+	rowFor := func(i int) []renderer.Cell {
	102	+	row := historyRow("abcd")
	103	+	row[i%4].Payload = payload
	104	+	return row
	105	+	}
	106	+	// The byte budget evicts from the tail (ChunkRows is large, so it never
	107	+	// seals); the 400-cell preallocation leaves room for many evictions in
	108	+	// one backing array before it would be reallocated.
	109	+	h := NewHistory(HistoryConfig{MaxRows: 100, MaxBytes: 1500, ChunkRows: 100})
	110	+	require.NoError(t, h.Append(rowFor(0), LineBound{End: 4}))
	111	+	orig := h.tailCells[:cap(h.tailCells)] // full original backing array
	112	+	evicted := 0
	113	+	for i := 1; i < 40 && len(h.chunks) == 0; i++ {
	114	+	before := h.Len()
	115	+	require.NoError(t, h.Append(rowFor(i), LineBound{End: 4}))
	116	+	if cap(h.tailCells) > len(orig) \|\| len(h.tailCells) == 0 {
	117	+	break
	118	+	}
	119	+	liveStart := len(orig) - cap(h.tailCells)
	120	+	if &orig[liveStart] != &h.tailCells[:1][0] {
	121	+	break // storage was reallocated; the old array is no longer ours
	122	+	}
	123	+	if h.Len() <= before {
	124	+	evicted++
	125	+	}
	126	+	liveEnd := liveStart + len(h.tailCells)
	127	+	for j, c := range orig {
	128	+	if j < liveStart \|\| j >= liveEnd {
	129	+	require.True(t, c.Payload.Empty(), "cell %d outside live window [%d,%d) pins payload", j, liveStart, liveEnd)
	130	+	}
	131	+	}
	132	+	}
	133	+	require.GreaterOrEqual(t, evicted, 2, "test did not exercise tail eviction")
	134	+	}
	135	+
	136	+	// checkHistoryAccounting compares the incrementally maintained LogicalBytes
	137	+	// with (1) the sum of the live chunks' own metadata and (2) an independent
	138	+	// recomputation: every live chunk is rebuilt from its decoded rows with a
	139	+	// fresh newHistoryChunks call (no recycled scratch state). Logical bytes depend
	140	+	// on the page layout, so the reference keeps each chunk's boundaries.
	141	+	func checkHistoryAccounting(t testing.T, h History) {
	142	+	t.Helper()
	143	+	view := h.View()
	144	+	require.Equal(t, historyChunksLogicalBytes(view.chunks), h.LogicalBytes(), "rows=%d", h.Len())
	145	+	require.Equal(t, h.Len(), view.Len())
	146	+	var rebuilt uint64
	147	+	for _, c := range view.chunks {
	148	+	rows := make([][]renderer.Cell, c.len())
	149	+	for i := range rows {
	150	+	rows[i] = c.row(i)
	151	+	}
	152	+	rebuilt += historyChunksLogicalBytes(newHistoryChunks(rows, c.bounds, c.rowIDs))
	153	+	}
	154	+	require.Equal(t, rebuilt, h.LogicalBytes(), "independent rebuild rows=%d", h.Len())
	155	+	}
	156	+
	157	+	// Accounting must stay exact after seals recycle the style and payload sets,
	158	+	// through eviction, SetLimits shrinking, payload rows, width changes and pages
	159	+	// with more than maxRetainedStyleScratch distinct styles.
	160	+	func TestHistoryLogicalBytesIndependentOfSetReuse(t *testing.T) {
	161	+	payload, err := renderer.NewCellPayload("x", "https://example.test")
	162	+	require.NoError(t, err)
	163	+	payload2, err := renderer.NewCellPayload("y", "")
	164	+	require.NoError(t, err)
	165	+	manyStyles := func(i int) []renderer.Cell { // 1100 distinct styles in one row
	166	+	row := make([]renderer.Cell, 1100)
	167	+	for x := range row {
	168	+	row[x] = renderer.Cell{Rune: 'z', Style: renderer.Style{HasForegroundRGB: true, ForegroundRGB: renderer.RGB{R: uint8(x), G: uint8(x >> 8), B: uint8(i)}}}
	169	+	}
	170	+	return row
	171	+	}
	172	+	rowFor := func(i int) []renderer.Cell {
	173	+	switch {
	174	+	case i%11 == 5:
	175	+	return manyStyles(i)
	176	+	case i%4 == 0:
	177	+	row := styledRow("abcd", i%7+1)
	178	+	row[1].Payload = payload
	179	+	row[2].Payload = payload2
	180	+	return row
	181	+	case i%3 == 0:
	182	+	return styledRow("abcdef", i%5+1) // width change forces a new page
	183	+	}
	184	+	return styledRow("abcd", i%7+1)
	185	+	}
	186	+	for _, tc := range []struct {
	187	+	name string
	188	+	cfg HistoryConfig
	189	+	}{
	190	+	{"non-divisible chunks", HistoryConfig{MaxRows: 200, MaxBytes: 1 << 30, ChunkRows: 7}},
	191	+	{"row eviction", HistoryConfig{MaxRows: 9, MaxBytes: 1 << 30, ChunkRows: 4}},
	192	+	{"byte eviction", HistoryConfig{MaxRows: 200, MaxBytes: 30 << 10, ChunkRows: 5}},
	193	+	{"tiny bytes", HistoryConfig{MaxRows: 200, MaxBytes: 1500, ChunkRows: 3}},
	194	+	} {
	195	+	t.Run(tc.name, func(t *testing.T) {
	196	+	h := NewHistory(tc.cfg)
	197	+	appendAt := func(i int) {
	198	+	row := rowFor(i)
	199	+	if err := h.Append(row, LineBound{End: len(row)}); err != nil {
	200	+	require.ErrorIs(t, err, ErrHistoryRowTooLarge)
	201	+	}
	202	+	checkHistoryAccounting(t, h)
	203	+	}
	204	+	for i := range 60 {
	205	+	appendAt(i)
	206	+	}
	207	+	shrunk := HistoryConfig{MaxRows: 5, MaxBytes: tc.cfg.MaxBytes, ChunkRows: 2}
	208	+	require.NoError(t, h.SetLimits(shrunk))
	209	+	checkHistoryAccounting(t, h)
	210	+	for i := 60; i < 75; i++ {
	211	+	appendAt(i)
	212	+	}
	213	+	})
	214	+	}
	215	+	}
	216	+
	217	+	// rowIDSet must accept and reject exactly what a plain map would, including
	218	+	// out-of-order IDs and duplicates across commits.
	219	+	func TestRowIDSetMatchesMap(t *testing.T) {
	220	+	rng := rand.New(rand.NewSource(1))
	221	+	for iter := 0; iter < 5000; iter++ {
	222	+	var set rowIDSet
	223	+	ref := map[RowID]struct{}{}
	224	+	for commit := 0; commit < 1+rng.Intn(4); commit++ {
	225	+	n := rng.Intn(6)
	226	+	ids := make([]RowID, n)
	227	+	base := RowID(rng.Intn(30))
	228	+	for i := range ids {
	229	+	if rng.Intn(3) == 0 {
	230	+	ids[i] = RowID(1 + rng.Intn(30))
	231	+	} else {
	232	+	ids[i] = base + RowID(i) + 1
	233	+	}
	234	+	}
	235	+	if rng.Intn(4) == 0 && n > 1 {
	236	+	ids[0], ids[n-1] = ids[n-1], ids[0]
	237	+	}
	238	+	want := true
	239	+	seen := map[RowID]struct{}{}
	240	+	for _, id := range ids {
	241	+	if _, dup := ref[id]; dup {
	242	+	want = false
	243	+	}
	244	+	if _, dup := seen[id]; dup {
	245	+	want = false
	246	+	}
	247	+	seen[id] = struct{}{}
	248	+	}
	249	+	set.reserve(n)
	250	+	for _, id := range ids {
	251	+	set.collect(id)
	252	+	}
	253	+	require.Equal(t, want, set.commit(), "iter %d ids %v", iter, ids)
	254	+	if !want {
	255	+	break
	256	+	}
	257	+	for _, id := range ids {
	258	+	ref[id] = struct{}{}
	259	+	}
	260	+	}
	261	+	}
	262	+	}
	263	+
	264	+	func TestRestoreAcceptsOutOfOrderDistinctRowIDsAcrossBlobs(t *testing.T) {
	265	+	src := NewHistory(HistoryConfig{MaxRows: 16, ChunkRows: 2})
	266	+	for _, id := range []RowID{100, 7, 8, 9} {
	267	+	require.NoError(t, src.AppendWithID(historyRow("abcd"), LineBound{End: 4}, id))
	268	+	}
	269	+	sealed, tail, err := MarshalSealedHistory(src.SealAndView())
	270	+	require.NoError(t, err)
	271	+	restored, err := HistoryFromBlobs(HistoryConfig{MaxRows: 16, ChunkRows: 2}, sealed, tail)
	272	+	require.NoError(t, err)
	273	+	require.Equal(t, 4, restored.Len())
	274	+	// Same IDs in two blobs are still rejected, in either order.
	275	+	_, err = HistoryFromBlobs(HistoryConfig{MaxRows: 16, ChunkRows: 2}, [][]byte{sealed[0], sealed[0]}, tail)
	276	+	require.Error(t, err)
	277	+	_, err = HistoryFromBlobs(HistoryConfig{MaxRows: 16, ChunkRows: 2}, [][]byte{sealed[1], sealed[0], sealed[1]}, tail)
	278	+	require.Error(t, err)
	279	+	}
	280	+
	281	+	// A chunk with more than maxRetainedStyleScratch styles must not leave its
	282	+	// large scratch map pinned, whether it is the last or an earlier chunk.
	283	+	func TestHistoryStyleScratchDroppedAfterHighCardinalityChunk(t *testing.T) {
	284	+	row := make([]renderer.Cell, 1100)
	285	+	for x := range row {
	286	+	row[x] = renderer.Cell{Rune: 'z', Style: renderer.Style{HasForegroundRGB: true, ForegroundRGB: renderer.RGB{R: uint8(x), G: uint8(x >> 8), B: 1}}}
	287	+	}
	288	+	h := NewHistory(HistoryConfig{MaxRows: 100, MaxBytes: 1 << 30, ChunkRows: 4})
	289	+	for range 4 { // seals one high-cardinality chunk
	290	+	require.NoError(t, h.Append(row, LineBound{End: len(row)}))
	291	+	}
	292	+	require.LessOrEqual(t, len(h.styleRowScratch), maxRetainedStyleScratch)
	293	+	require.LessOrEqual(t, len(h.tailPageStyles), maxRetainedStyleScratch+1)
	294	+	for range 3 {
	295	+	require.NoError(t, h.Append(historyRow("abcd"), LineBound{End: 4}))
	296	+	}
	297	+	_ = h.View()
	298	+	require.LessOrEqual(t, len(h.styleRowScratch), maxRetainedStyleScratch)
	299	+	}
	300	+
	301	+	// Early false returns from validation must not leak collected IDs into a later
	302	+	// commit.
	303	+	func TestRowIDSetBeginDiscardsUncommittedIDs(t *testing.T) {
	304	+	var set rowIDSet
	305	+	set.begin(0)
	306	+	set.collect(1)
	307	+	set.collect(2) // abandoned without commit, as on an early invalid return
	308	+	set.begin(0)
	309	+	set.collect(2)
	310	+	require.True(t, set.commit())
	311	+	set.begin(0)
	312	+	set.collect(1)
	313	+	require.True(t, set.commit())
	314	+	}

@@ -0,0 +1,314 @@
	1	+	package vt
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"math/rand"
	6	+	"testing"
	7	+
	8	+	renderer "github.com/bnema/vev-vt/core"
	9	+	"github.com/stretchr/testify/require"
	10	+	)
	11	+
	12	+	func styledRow(text string, fg int) []renderer.Cell {
	13	+	row := historyRow(text)
	14	+	for i := range row {
	15	+	row[i].Style = renderer.Style{Bold: true, Foreground: fg}
	16	+	}
	17	+	return row
	18	+	}
	19	+
	20	+	func rangeTexts(t *testing.T, v HistoryView) []string {
	21	+	t.Helper()
	22	+	var out []string
	23	+	require.NoError(t, v.Range(func(row []renderer.Cell) bool {
	24	+	out = append(out, rowText(row))
	25	+	return true
	26	+	}))
	27	+	return out
	28	+	}
	29	+
	30	+	// Sealing recycles the private tail storage. No previously handed-out view,
	31	+	// snapshot tail or Range row may observe later appends or seals.
	32	+	func TestHistoryTailStorageReuseKeepsViewsStable(t *testing.T) {
	33	+	h := NewHistory(HistoryConfig{MaxRows: 64, MaxBytes: 1 << 20, ChunkRows: 4})
	34	+	appendRow := func(i int) {
	35	+	require.NoError(t, h.Append(styledRow(fmt.Sprintf("r%03d", i), i%5+1), LineBound{End: 4}))
	36	+	}
	37	+	for i := range 2 {
	38	+	appendRow(i)
	39	+	}
	40	+	partial := h.View()
	41	+	snap := h.SnapshotView()
	42	+	var ranged [][]renderer.Cell
	43	+	require.NoError(t, partial.Range(func(row []renderer.Cell) bool { ranged = append(ranged, row); return true }))
	44	+
	45	+	for i := 2; i < 14; i++ { // seals three times, reusing storage each time
	46	+	appendRow(i)
	47	+	}
	48	+	sealedView := h.SealAndView()
	49	+	appendRow(14)
	50	+	appendRow(15)
	51	+
	52	+	require.Equal(t, []string{"r000", "r001"}, rangeTexts(t, partial))
	53	+	require.Equal(t, []string{"r000", "r001"}, rangeTexts(t, snap.Tail()))
	54	+	require.Equal(t, "r000", rowText(ranged[0]))
	55	+	require.Equal(t, "r001", rowText(ranged[1]))
	56	+	require.Equal(t, 14, sealedView.Len())
	57	+	texts := rangeTexts(t, sealedView)
	58	+	for i, got := range texts {
	59	+	require.Equal(t, fmt.Sprintf("r%03d", i), got)
	60	+	}
	61	+	live := rangeTexts(t, h.View())
	62	+	require.Len(t, live, 16)
	63	+	require.Equal(t, "r015", live[15])
	64	+	for _, c := range h.chunks {
	65	+	require.NoError(t, c.CheckInvariants())
	66	+	}
	67	+	}
	68	+
	69	+	func TestHistoryTailStorageIsRecycledAcrossSeals(t *testing.T) {
	70	+	h := NewHistory(HistoryConfig{MaxRows: 64, MaxBytes: 1 << 20, ChunkRows: 4})
	71	+	for i := range 4 {
	72	+	require.NoError(t, h.Append(historyRow(fmt.Sprintf("r%03d", i)), LineBound{End: 4}))
	73	+	}
	74	+	require.Empty(t, h.tail)
	75	+	require.Zero(t, len(h.tailCells))
	76	+	require.NotZero(t, cap(h.tailCells), "seal must keep tail storage for the next chunk")
	77	+	first := &h.tailCells[:1][0]
	78	+	for i := 4; i < 8; i++ {
	79	+	require.NoError(t, h.Append(historyRow(fmt.Sprintf("r%03d", i)), LineBound{End: 4}))
	80	+	}
	81	+	require.Same(t, first, &h.tailCells[:1][0], "storage reallocated across seals")
	82	+	// A reused backing array must not pin payload strings of sealed rows.
	83	+	payload, err := renderer.NewCellPayload("é\u0301", "https://example.test")
	84	+	require.NoError(t, err)
	85	+	row := historyRow("abcd")
	86	+	row[0].Payload = payload
	87	+	for range 4 {
	88	+	require.NoError(t, h.Append(row, LineBound{End: 4}))
	89	+	}
	90	+	for _, c := range h.tailCells[:cap(h.tailCells)] {
	91	+	require.True(t, c.Payload.Empty())
	92	+	}
	93	+	require.NoError(t, h.chunks[len(h.chunks)-1].CheckInvariants())
	94	+	}
	95	+
	96	+	// Eviction from a partial tail must not leave the evicted rows' payload
	97	+	// strings pinned in the recycled backing array's prefix.
	98	+	func TestHistoryTailEvictionClearsEvictedPrefix(t *testing.T) {
	99	+	payload, err := renderer.NewCellPayload("é\u0301", "https://example.test")
	100	+	require.NoError(t, err)
	101	+	rowFor := func(i int) []renderer.Cell {
	102	+	row := historyRow("abcd")
	103	+	row[i%4].Payload = payload
	104	+	return row
	105	+	}
	106	+	// The byte budget evicts from the tail (ChunkRows is large, so it never
	107	+	// seals); the 400-cell preallocation leaves room for many evictions in
	108	+	// one backing array before it would be reallocated.
	109	+	h := NewHistory(HistoryConfig{MaxRows: 100, MaxBytes: 1500, ChunkRows: 100})
	110	+	require.NoError(t, h.Append(rowFor(0), LineBound{End: 4}))
	111	+	orig := h.tailCells[:cap(h.tailCells)] // full original backing array
	112	+	evicted := 0
	113	+	for i := 1; i < 40 && len(h.chunks) == 0; i++ {
	114	+	before := h.Len()
	115	+	require.NoError(t, h.Append(rowFor(i), LineBound{End: 4}))
	116	+	if cap(h.tailCells) > len(orig) \|\| len(h.tailCells) == 0 {
	117	+	break
	118	+	}
	119	+	liveStart := len(orig) - cap(h.tailCells)
	120	+	if &orig[liveStart] != &h.tailCells[:1][0] {
	121	+	break // storage was reallocated; the old array is no longer ours
	122	+	}
	123	+	if h.Len() <= before {
	124	+	evicted++
	125	+	}
	126	+	liveEnd := liveStart + len(h.tailCells)
	127	+	for j, c := range orig {
	128	+	if j < liveStart \|\| j >= liveEnd {
	129	+	require.True(t, c.Payload.Empty(), "cell %d outside live window [%d,%d) pins payload", j, liveStart, liveEnd)
	130	+	}
	131	+	}
	132	+	}
	133	+	require.GreaterOrEqual(t, evicted, 2, "test did not exercise tail eviction")
	134	+	}
	135	+
	136	+	// checkHistoryAccounting compares the incrementally maintained LogicalBytes
	137	+	// with (1) the sum of the live chunks' own metadata and (2) an independent
	138	+	// recomputation: every live chunk is rebuilt from its decoded rows with a
	139	+	// fresh newHistoryChunks call (no recycled scratch state). Logical bytes depend
	140	+	// on the page layout, so the reference keeps each chunk's boundaries.
	141	+	func checkHistoryAccounting(t testing.T, h History) {
	142	+	t.Helper()
	143	+	view := h.View()
	144	+	require.Equal(t, historyChunksLogicalBytes(view.chunks), h.LogicalBytes(), "rows=%d", h.Len())
	145	+	require.Equal(t, h.Len(), view.Len())
	146	+	var rebuilt uint64
	147	+	for _, c := range view.chunks {
	148	+	rows := make([][]renderer.Cell, c.len())
	149	+	for i := range rows {
	150	+	rows[i] = c.row(i)
	151	+	}
	152	+	rebuilt += historyChunksLogicalBytes(newHistoryChunks(rows, c.bounds, c.rowIDs))
	153	+	}
	154	+	require.Equal(t, rebuilt, h.LogicalBytes(), "independent rebuild rows=%d", h.Len())
	155	+	}
	156	+
	157	+	// Accounting must stay exact after seals recycle the style and payload sets,
	158	+	// through eviction, SetLimits shrinking, payload rows, width changes and pages
	159	+	// with more than maxRetainedStyleScratch distinct styles.
	160	+	func TestHistoryLogicalBytesIndependentOfSetReuse(t *testing.T) {
	161	+	payload, err := renderer.NewCellPayload("x", "https://example.test")
	162	+	require.NoError(t, err)
	163	+	payload2, err := renderer.NewCellPayload("y", "")
	164	+	require.NoError(t, err)
	165	+	manyStyles := func(i int) []renderer.Cell { // 1100 distinct styles in one row
	166	+	row := make([]renderer.Cell, 1100)
	167	+	for x := range row {
	168	+	row[x] = renderer.Cell{Rune: 'z', Style: renderer.Style{HasForegroundRGB: true, ForegroundRGB: renderer.RGB{R: uint8(x), G: uint8(x >> 8), B: uint8(i)}}}
	169	+	}
	170	+	return row
	171	+	}
	172	+	rowFor := func(i int) []renderer.Cell {
	173	+	switch {
	174	+	case i%11 == 5:
	175	+	return manyStyles(i)
	176	+	case i%4 == 0:
	177	+	row := styledRow("abcd", i%7+1)
	178	+	row[1].Payload = payload
	179	+	row[2].Payload = payload2
	180	+	return row
	181	+	case i%3 == 0:
	182	+	return styledRow("abcdef", i%5+1) // width change forces a new page
	183	+	}
	184	+	return styledRow("abcd", i%7+1)
	185	+	}
	186	+	for _, tc := range []struct {
	187	+	name string
	188	+	cfg HistoryConfig
	189	+	}{
	190	+	{"non-divisible chunks", HistoryConfig{MaxRows: 200, MaxBytes: 1 << 30, ChunkRows: 7}},
	191	+	{"row eviction", HistoryConfig{MaxRows: 9, MaxBytes: 1 << 30, ChunkRows: 4}},
	192	+	{"byte eviction", HistoryConfig{MaxRows: 200, MaxBytes: 30 << 10, ChunkRows: 5}},
	193	+	{"tiny bytes", HistoryConfig{MaxRows: 200, MaxBytes: 1500, ChunkRows: 3}},
	194	+	} {
	195	+	t.Run(tc.name, func(t *testing.T) {
	196	+	h := NewHistory(tc.cfg)
	197	+	appendAt := func(i int) {
	198	+	row := rowFor(i)
	199	+	if err := h.Append(row, LineBound{End: len(row)}); err != nil {
	200	+	require.ErrorIs(t, err, ErrHistoryRowTooLarge)
	201	+	}
	202	+	checkHistoryAccounting(t, h)
	203	+	}
	204	+	for i := range 60 {
	205	+	appendAt(i)
	206	+	}
	207	+	shrunk := HistoryConfig{MaxRows: 5, MaxBytes: tc.cfg.MaxBytes, ChunkRows: 2}
	208	+	require.NoError(t, h.SetLimits(shrunk))
	209	+	checkHistoryAccounting(t, h)
	210	+	for i := 60; i < 75; i++ {
	211	+	appendAt(i)
	212	+	}
	213	+	})
	214	+	}
	215	+	}
	216	+
	217	+	// rowIDSet must accept and reject exactly what a plain map would, including
	218	+	// out-of-order IDs and duplicates across commits.
	219	+	func TestRowIDSetMatchesMap(t *testing.T) {
	220	+	rng := rand.New(rand.NewSource(1))
	221	+	for iter := 0; iter < 5000; iter++ {
	222	+	var set rowIDSet
	223	+	ref := map[RowID]struct{}{}
	224	+	for commit := 0; commit < 1+rng.Intn(4); commit++ {
	225	+	n := rng.Intn(6)
	226	+	ids := make([]RowID, n)
	227	+	base := RowID(rng.Intn(30))
	228	+	for i := range ids {
	229	+	if rng.Intn(3) == 0 {
	230	+	ids[i] = RowID(1 + rng.Intn(30))
	231	+	} else {
	232	+	ids[i] = base + RowID(i) + 1
	233	+	}
	234	+	}
	235	+	if rng.Intn(4) == 0 && n > 1 {
	236	+	ids[0], ids[n-1] = ids[n-1], ids[0]
	237	+	}
	238	+	want := true
	239	+	seen := map[RowID]struct{}{}
	240	+	for _, id := range ids {
	241	+	if _, dup := ref[id]; dup {
	242	+	want = false
	243	+	}
	244	+	if _, dup := seen[id]; dup {
	245	+	want = false
	246	+	}
	247	+	seen[id] = struct{}{}
	248	+	}
	249	+	set.reserve(n)
	250	+	for _, id := range ids {
	251	+	set.collect(id)
	252	+	}
	253	+	require.Equal(t, want, set.commit(), "iter %d ids %v", iter, ids)
	254	+	if !want {
	255	+	break
	256	+	}
	257	+	for _, id := range ids {
	258	+	ref[id] = struct{}{}
	259	+	}
	260	+	}
	261	+	}
	262	+	}
	263	+
	264	+	func TestRestoreAcceptsOutOfOrderDistinctRowIDsAcrossBlobs(t *testing.T) {
	265	+	src := NewHistory(HistoryConfig{MaxRows: 16, ChunkRows: 2})
	266	+	for _, id := range []RowID{100, 7, 8, 9} {
	267	+	require.NoError(t, src.AppendWithID(historyRow("abcd"), LineBound{End: 4}, id))
	268	+	}
	269	+	sealed, tail, err := MarshalSealedHistory(src.SealAndView())
	270	+	require.NoError(t, err)
	271	+	restored, err := HistoryFromBlobs(HistoryConfig{MaxRows: 16, ChunkRows: 2}, sealed, tail)
	272	+	require.NoError(t, err)
	273	+	require.Equal(t, 4, restored.Len())
	274	+	// Same IDs in two blobs are still rejected, in either order.
	275	+	_, err = HistoryFromBlobs(HistoryConfig{MaxRows: 16, ChunkRows: 2}, [][]byte{sealed[0], sealed[0]}, tail)
	276	+	require.Error(t, err)
	277	+	_, err = HistoryFromBlobs(HistoryConfig{MaxRows: 16, ChunkRows: 2}, [][]byte{sealed[1], sealed[0], sealed[1]}, tail)
	278	+	require.Error(t, err)
	279	+	}
	280	+
	281	+	// A chunk with more than maxRetainedStyleScratch styles must not leave its
	282	+	// large scratch map pinned, whether it is the last or an earlier chunk.
	283	+	func TestHistoryStyleScratchDroppedAfterHighCardinalityChunk(t *testing.T) {
	284	+	row := make([]renderer.Cell, 1100)
	285	+	for x := range row {
	286	+	row[x] = renderer.Cell{Rune: 'z', Style: renderer.Style{HasForegroundRGB: true, ForegroundRGB: renderer.RGB{R: uint8(x), G: uint8(x >> 8), B: 1}}}
	287	+	}
	288	+	h := NewHistory(HistoryConfig{MaxRows: 100, MaxBytes: 1 << 30, ChunkRows: 4})
	289	+	for range 4 { // seals one high-cardinality chunk
	290	+	require.NoError(t, h.Append(row, LineBound{End: len(row)}))
	291	+	}
	292	+	require.LessOrEqual(t, len(h.styleRowScratch), maxRetainedStyleScratch)
	293	+	require.LessOrEqual(t, len(h.tailPageStyles), maxRetainedStyleScratch+1)
	294	+	for range 3 {
	295	+	require.NoError(t, h.Append(historyRow("abcd"), LineBound{End: 4}))
	296	+	}
	297	+	_ = h.View()
	298	+	require.LessOrEqual(t, len(h.styleRowScratch), maxRetainedStyleScratch)
	299	+	}
	300	+
	301	+	// Early false returns from validation must not leak collected IDs into a later
	302	+	// commit.
	303	+	func TestRowIDSetBeginDiscardsUncommittedIDs(t *testing.T) {
	304	+	var set rowIDSet
	305	+	set.begin(0)
	306	+	set.collect(1)
	307	+	set.collect(2) // abandoned without commit, as on an early invalid return
	308	+	set.begin(0)
	309	+	set.collect(2)
	310	+	require.True(t, set.commit())
	311	+	set.begin(0)
	312	+	set.collect(1)
	313	+	require.True(t, set.commit())
	314	+	}

M snapshot.go +9 -10


@@ -88,22 +88,22 @@ func HistoryFromBlobs(config HistoryConfig, sealed [][]byte, tail []byte) (*Hist
88	88		}
89	89
90	90		func decodeRestoredHistoryBlobs(sealed [][]byte, tail []byte, extra ...HistoryView) ([]HistoryView, HistoryView, error) {
91		-	seen := make(map[RowID]struct{})
	91	+	var seen rowIDSet
92	92		sealedViews := make([]HistoryView, len(sealed))
93	93		for i, blob := range sealed {
94	94		view, err := UnmarshalHistory(blob)
95		-	if err != nil \|\| len(view.chunks) != 1 \|\| view.chunks[0].len() == 0 \|\| !validateRestoredHistoryView(view, seen) {
	95	+	if err != nil \|\| len(view.chunks) != 1 \|\| view.chunks[0].len() == 0 \|\| !validateRestoredHistoryView(view, &seen) {
96	96		return nil, HistoryView{}, fmt.Errorf("restore sealed history: %w", errInvalidHistory)
97	97		}
98	98		sealedViews[i] = view
99	99		}
100	100
101	101		tailView, err := UnmarshalHistory(tail)
102		-	if err != nil \|\| !validateRestoredHistoryView(tailView, seen) {
	102	+	if err != nil \|\| !validateRestoredHistoryView(tailView, &seen) {
103	103		return nil, HistoryView{}, fmt.Errorf("restore history tail: %w", errInvalidHistory)
104	104		}
105	105		for _, view := range extra {
106		-	if !validateRestoredHistoryView(view, seen) {
	106	+	if !validateRestoredHistoryView(view, &seen) {
107	107		return nil, HistoryView{}, fmt.Errorf("restore recovery transcript: %w", errInvalidHistory)
108	108		}
109	109		}

@@ -174,27 +174,26 @@ func (h History) boundedRestoredChunk(chunk HistoryChunk) *HistoryChunk {
174	174		return bounded
175	175		}
176	176
177		-	func validateRestoredHistoryView(view HistoryView, seen map[RowID]struct{}) bool {
	177	+	func validateRestoredHistoryView(view HistoryView, seen *rowIDSet) bool {
178	178		if view.nextRowID == 0 \|\| view.nextRowID == ^RowID(0) {
179	179		return false
180	180		}
181	181		var maxID RowID
	182	+	seen.begin(0)
182	183		for _, chunk := range view.chunks {
183	184		if chunk == nil \|\| chunk.len() != len(chunk.rowIDs) \|\| chunk.len() != len(chunk.bounds) {
184	185		return false
185	186		}
	187	+	seen.reserve(len(chunk.rowIDs))
186	188		for _, id := range chunk.rowIDs {
187	189		if id == 0 \|\| id >= ^RowID(0)-1 {
188	190		return false
189	191		}
190		-	if _, duplicate := seen[id]; duplicate {
191		-	return false
192		-	}
193		-	seen[id] = struct{}{}
	192	+	seen.collect(id)
194	193		maxID = max(maxID, id)
195	194		}
196	195		}
197		-	return view.nextRowID > maxID
	196	+	return view.nextRowID > maxID && seen.commit()
198	197		}
199	198
200	199		// NewScreenWithRecoveryTranscript constructs a fresh blank screen whose

 }
 func decodeRestoredHistoryBlobs(sealed [][]byte, tail []byte, extra ...HistoryView) ([]HistoryView, HistoryView, error) {
-	seen := make(map[RowID]struct{})
+	var seen rowIDSet
 	sealedViews := make([]HistoryView, len(sealed))
 	for i, blob := range sealed {
 		view, err := UnmarshalHistory(blob)
-		if err != nil || len(view.chunks) != 1 || view.chunks[0].len() == 0 || !validateRestoredHistoryView(view, seen) {
+		if err != nil || len(view.chunks) != 1 || view.chunks[0].len() == 0 || !validateRestoredHistoryView(view, &seen) {
 			return nil, HistoryView{}, fmt.Errorf("restore sealed history: %w", errInvalidHistory)
 		}
 		sealedViews[i] = view
 	}
 	tailView, err := UnmarshalHistory(tail)
-	if err != nil || !validateRestoredHistoryView(tailView, seen) {
+	if err != nil || !validateRestoredHistoryView(tailView, &seen) {
 		return nil, HistoryView{}, fmt.Errorf("restore history tail: %w", errInvalidHistory)
 	}
 	for _, view := range extra {
-		if !validateRestoredHistoryView(view, seen) {
+		if !validateRestoredHistoryView(view, &seen) {
 			return nil, HistoryView{}, fmt.Errorf("restore recovery transcript: %w", errInvalidHistory)
 		}
 	}
 	return bounded
 }
-func validateRestoredHistoryView(view HistoryView, seen map[RowID]struct{}) bool {
+func validateRestoredHistoryView(view HistoryView, seen *rowIDSet) bool {
 	if view.nextRowID == 0 || view.nextRowID == ^RowID(0) {
 		return false
 	}
 	var maxID RowID
+	seen.begin(0)
 	for _, chunk := range view.chunks {
 		if chunk == nil || chunk.len() != len(chunk.rowIDs) || chunk.len() != len(chunk.bounds) {
 			return false
 		}
+		seen.reserve(len(chunk.rowIDs))
 		for _, id := range chunk.rowIDs {
 			if id == 0 || id >= ^RowID(0)-1 {
 				return false
 			}
-			if _, duplicate := seen[id]; duplicate {
+			seen.collect(id)
-				return false
-			}
-			seen[id] = struct{}{}
 			maxID = max(maxID, id)
 		}
 	}
-	return view.nextRowID > maxID
+	return view.nextRowID > maxID && seen.commit()
 }
 // NewScreenWithRecoveryTranscript constructs a fresh blank screen whose