Merge pull request #216 from m1k3s0/fix/extended-color-params

Fix mangled extended color sequences in syntax highlighting

commit: 41873f6fa950af67d31743e0b4d54f6c8d827cd1
author: Patric Eckhart · 25d ago · 2026-09-27 08:04 UTC
committer: GitHub · 25d ago · 2026-09-27 08:04 UTC
merge of: f2fef94 + cdd2065 · diff against the first parent
signature: GPG UnverifiedB5690EEEBB952194 The key is unknown

2 files · +191 -10

M packages/tui/highlight.go +42 -10


@@ -121,6 +121,14 @@ func LanguageFromPath(p string) string {
121	121		// Chroma's inherited styles can assign black backgrounds to a few tokens
122	122		// (notably punctuation/error-ish spans), which looks like random black blocks
123	123		// inside zot's dark-gray tool boxes. Foreground color is enough for code.
	124	+	//
	125	+	// Extended color parameters ("38;5;N", "38;2;R;G;B" and their "48;..."
	126	+	// background twins) are consumed as whole units. The color index of an
	127	+	// extended foreground is a value, not a standalone ANSI code: stripping a
	128	+	// "45" or "101" out of "\x1b[38;5;45m" leaves the malformed "\x1b[38;5m",
	129	+	// whose leftover "5" terminals read as blink (SGR 5), so the token loses its
	130	+	// color and blinks. A truncated extended color cannot be rendered anyway,
	131	+	// so it is dropped rather than forwarded.
124	132		func stripANSIBackgrounds(s string) string {
125	133		var out strings.Builder
126	134		out.Grow(len(s))

@@ -143,21 +151,21 @@ func stripANSIBackgrounds(s string) string {
143	151		for p := 0; p < len(params); p++ {
144	152		param := params[p]
145	153		switch param {
146		-	case "48":
147		-	// 48;5;N or 48;2;R;G;B background color.
148		-	if p+1 < len(params) && params[p+1] == "5" {
149		-	p += 2
150		-	continue
151		-	}
152		-	if p+1 < len(params) && params[p+1] == "2" {
153		-	p += 4
154		-	continue
	154	+	case "38", "48":
	155	+	span, complete := extendedColorSpan(params, p)
	156	+	if complete && param == "38" {
	157	+	kept = append(kept, span...)
155	158		}
	159	+	// Consume the whole unit either way: a truncated
	160	+	// "38;5" must not leave its "5" to be read as blink.
	161	+	p += len(span) - 1
156	162		continue
157	163		case "49":
158	164		continue
159	165		}
160		-	// 40-47 and 100-107 are ANSI background colors.
	166	+	// 40-47 and 100-107 are ANSI background colors. Bare codes only:
	167	+	// extended colors were consumed above, so a color index in those
	168	+	// ranges cannot be mistaken for one here.
161	169		if len(param) == 2 && param[0] == '4' && param[1] >= '0' && param[1] <= '7' {
162	170		continue
163	171		}

@@ -176,6 +184,30 @@ func stripANSIBackgrounds(s string) string {
176	184		return out.String()
177	185		}
178	186
	187	+	// extendedColorSpan returns the parameter run that belongs to the extended
	188	+	// color starting at params[i] ("38" or "48"): "5;N" or "2;R;G;B". complete
	189	+	// reports whether the run can be rendered; a truncated run is still returned
	190	+	// so its parameters are consumed instead of being reprocessed as standalone
	191	+	// codes (a bare "5" would mean blink).
	192	+	func extendedColorSpan(params []string, i int) (span []string, complete bool) {
	193	+	if i+1 >= len(params) {
	194	+	return params[i : i+1], false
	195	+	}
	196	+	switch params[i+1] {
	197	+	case "5":
	198	+	if i+2 >= len(params) {
	199	+	return params[i : i+2], false
	200	+	}
	201	+	return params[i : i+3], true
	202	+	case "2":
	203	+	if i+5 > len(params) {
	204	+	return params[i:], false
	205	+	}
	206	+	return params[i : i+5], true
	207	+	}
	208	+	return params[i : i+1], false
	209	+	}
	210	+
179	211		func chooseLexer(lang string) chroma.Lexer {
180	212		if lang == "" {
181	213		return nil


@@ -121,6 +121,14 @@ func LanguageFromPath(p string) string {
121		// Chroma's inherited styles can assign black backgrounds to a few tokens	121		// Chroma's inherited styles can assign black backgrounds to a few tokens
122		// (notably punctuation/error-ish spans), which looks like random black blocks	122		// (notably punctuation/error-ish spans), which looks like random black blocks
123		// inside zot's dark-gray tool boxes. Foreground color is enough for code.	123		// inside zot's dark-gray tool boxes. Foreground color is enough for code.
			124	+	//
			125	+	// Extended color parameters ("38;5;N", "38;2;R;G;B" and their "48;..."
			126	+	// background twins) are consumed as whole units. The color index of an
			127	+	// extended foreground is a value, not a standalone ANSI code: stripping a
			128	+	// "45" or "101" out of "\x1b[38;5;45m" leaves the malformed "\x1b[38;5m",
			129	+	// whose leftover "5" terminals read as blink (SGR 5), so the token loses its
			130	+	// color and blinks. A truncated extended color cannot be rendered anyway,
			131	+	// so it is dropped rather than forwarded.
124		func stripANSIBackgrounds(s string) string {	132		func stripANSIBackgrounds(s string) string {
125		var out strings.Builder	133		var out strings.Builder
126		out.Grow(len(s))	134		out.Grow(len(s))

@@ -143,21 +151,21 @@ func stripANSIBackgrounds(s string) string {
143		for p := 0; p < len(params); p++ {	151		for p := 0; p < len(params); p++ {
144		param := params[p]	152		param := params[p]
145		switch param {	153		switch param {
146	-	case "48":	154	+	case "38", "48":
147	-	// 48;5;N or 48;2;R;G;B background color.	155	+	span, complete := extendedColorSpan(params, p)
148	-	if p+1 < len(params) && params[p+1] == "5" {	156	+	if complete && param == "38" {
149	-	p += 2	157	+	kept = append(kept, span...)
150	-	continue
151	-	}
152	-	if p+1 < len(params) && params[p+1] == "2" {
153	-	p += 4
154	-	continue
155		}	158		}
			159	+	// Consume the whole unit either way: a truncated
			160	+	// "38;5" must not leave its "5" to be read as blink.
			161	+	p += len(span) - 1
156		continue	162		continue
157		case "49":	163		case "49":
158		continue	164		continue
159		}	165		}
160	-	// 40-47 and 100-107 are ANSI background colors.	166	+	// 40-47 and 100-107 are ANSI background colors. Bare codes only:
			167	+	// extended colors were consumed above, so a color index in those
			168	+	// ranges cannot be mistaken for one here.
161		if len(param) == 2 && param[0] == '4' && param[1] >= '0' && param[1] <= '7' {	169		if len(param) == 2 && param[0] == '4' && param[1] >= '0' && param[1] <= '7' {
162		continue	170		continue
163		}	171		}

@@ -176,6 +184,30 @@ func stripANSIBackgrounds(s string) string {
176		return out.String()	184		return out.String()
177		}	185		}
178			186
			187	+	// extendedColorSpan returns the parameter run that belongs to the extended
			188	+	// color starting at params[i] ("38" or "48"): "5;N" or "2;R;G;B". complete
			189	+	// reports whether the run can be rendered; a truncated run is still returned
			190	+	// so its parameters are consumed instead of being reprocessed as standalone
			191	+	// codes (a bare "5" would mean blink).
			192	+	func extendedColorSpan(params []string, i int) (span []string, complete bool) {
			193	+	if i+1 >= len(params) {
			194	+	return params[i : i+1], false
			195	+	}
			196	+	switch params[i+1] {
			197	+	case "5":
			198	+	if i+2 >= len(params) {
			199	+	return params[i : i+2], false
			200	+	}
			201	+	return params[i : i+3], true
			202	+	case "2":
			203	+	if i+5 > len(params) {
			204	+	return params[i:], false
			205	+	}
			206	+	return params[i : i+5], true
			207	+	}
			208	+	return params[i : i+1], false
			209	+	}
			210	+
179		func chooseLexer(lang string) chroma.Lexer {	211		func chooseLexer(lang string) chroma.Lexer {
180		if lang == "" {	212		if lang == "" {
181		return nil	213		return nil

A packages/tui/highlight_test.go +149 -0

@@ -0,0 +1,149 @@
	1	+	package tui
	2	+
	3	+	import (
	4	+	"regexp"
	5	+	"strings"
	6	+	"testing"
	7	+	)
	8	+
	9	+	// The stripper must treat "38;5;N" / "38;2;R;G;B" as one unit. Pulling the
	10	+	// color index out on its own (when it happens to look like an ANSI
	11	+	// background code, e.g. 45, 48, 49 or 101) leaves "\x1b[38;5m", whose leftover
	12	+	// "5" terminals read as blink, so the token loses its color and blinks.
	13	+	func TestStripANSIBackgroundsKeepsExtendedColors(t *testing.T) {
	14	+	cases := []struct{ name, in, want string }{
	15	+	{"256 fg in background range", "\x1b[38;5;45mstring\x1b[0m", "\x1b[38;5;45mstring\x1b[0m"},
	16	+	{"256 fg index 48", "\x1b[38;5;48mx\x1b[0m", "\x1b[38;5;48mx\x1b[0m"},
	17	+	{"256 fg index 49", "\x1b[38;5;49mx\x1b[0m", "\x1b[38;5;49mx\x1b[0m"},
	18	+	{"256 fg bright range", "\x1b[38;5;101mcomment\x1b[0m", "\x1b[38;5;101mcomment\x1b[0m"},
	19	+	{"256 fg index 107", "\x1b[38;5;107mx\x1b[0m", "\x1b[38;5;107mx\x1b[0m"},
	20	+	{"256 fg index 41", "\x1b[38;5;41mx\x1b[0m", "\x1b[38;5;41mx\x1b[0m"},
	21	+	{"bold plus 256 fg", "\x1b[1;38;5;45mstring\x1b[0m", "\x1b[1;38;5;45mstring\x1b[0m"},
	22	+	{"truecolor fg", "\x1b[38;2;0;229;255mstring\x1b[0m", "\x1b[38;2;0;229;255mstring\x1b[0m"},
	23	+	{"plain text", "hello", "hello"},
	24	+	{"reset", "\x1b[0m", "\x1b[0m"},
	25	+
	26	+	{"256 bg dropped", "\x1b[48;5;45mbg\x1b[0m", "bg\x1b[0m"},
	27	+	{"256 bg dropped, fg kept", "\x1b[38;5;45m\x1b[48;5;0mtext", "\x1b[38;5;45mtext"},
	28	+	{"truecolor bg dropped", "\x1b[48;2;9;0;21mbg\x1b[0m", "bg\x1b[0m"},
	29	+	{"default bg dropped", "\x1b[49mbg", "bg"},
	30	+	{"ansi bg 41 dropped", "\x1b[41mbg", "bg"},
	31	+	{"ansi bg 101 dropped", "\x1b[101mbg", "bg"},
	32	+	{"truncated fg dropped", "\x1b[38;5mstring", "string"},
	33	+	{"truncated bg dropped", "\x1b[48;5mstring", "string"},
	34	+	{"truncated fg inside row", "\x1b[1m\x1b[38;5;109mfunc\x1b[0m\x1b[38;5m(", "\x1b[1m\x1b[38;5;109mfunc\x1b[0m("},
	35	+	}
	36	+	for _, c := range cases {
	37	+	if got := stripANSIBackgrounds(c.in); got != c.want {
	38	+	t.Errorf("%s:\n in %q\n got %q\n want %q", c.name, c.in, got, c.want)
	39	+	}
	40	+	}
	41	+	}
	42	+
	43	+	// incompleteExtendedColor returns the first SGR sequence in s whose extended
	44	+	// color introducer ("38" or "48") is missing part of its parameter run, for
	45	+	// example "\x1b[38;5m", which terminals read as a color-less SGR 38 followed
	46	+	// by SGR 5 (blink). Validation here is independent of the production helper.
	47	+	func incompleteExtendedColor(s string) string {
	48	+	numeric := func(p string) bool {
	49	+	if p == "" {
	50	+	return false
	51	+	}
	52	+	for _, r := range p {
	53	+	if r < '0' \|\| r > '9' {
	54	+	return false
	55	+	}
	56	+	}
	57	+	return true
	58	+	}
	59	+	for i := 0; i < len(s); {
	60	+	if s[i] != 0x1b \|\| i+1 >= len(s) \|\| s[i+1] != '[' {
	61	+	i++
	62	+	continue
	63	+	}
	64	+	j := i + 2
	65	+	for j < len(s) && !(s[j] >= 0x40 && s[j] <= 0x7e) {
	66	+	j++
	67	+	}
	68	+	if j >= len(s) {
	69	+	return s[i:]
	70	+	}
	71	+	if s[j] == 'm' {
	72	+	params := strings.Split(s[i+2:j], ";")
	73	+	for p := 0; p < len(params); p++ {
	74	+	if params[p] != "38" && params[p] != "48" {
	75	+	continue
	76	+	}
	77	+	need := 0
	78	+	switch {
	79	+	case p+1 < len(params) && params[p+1] == "5":
	80	+	need = 2
	81	+	case p+1 < len(params) && params[p+1] == "2":
	82	+	need = 4
	83	+	default:
	84	+	return s[i : j+1]
	85	+	}
	86	+	if p+need >= len(params) \|\| !numeric(params[p+2]) {
	87	+	return s[i : j+1]
	88	+	}
	89	+	if need == 4 {
	90	+	for k := p + 2; k <= p+4 && k < len(params); k++ {
	91	+	if !numeric(params[k]) {
	92	+	return s[i : j+1]
	93	+	}
	94	+	}
	95	+	}
	96	+	p += need
	97	+	}
	98	+	}
	99	+	i = j + 1
	100	+	}
	101	+	return ""
	102	+	}
	103	+
	104	+	// A theme color that chroma maps to xterm index 45 ("#00e5ff") must survive
	105	+	// highlighting with a renderable sequence: a complete "\x1b[38;5;45m", never
	106	+	// the malformed "\x1b[38;5m".
	107	+	func TestHighlightCodeExtendedColorIsWellFormed(t *testing.T) {
	108	+	th := Dark
	109	+	th.Syntax = SyntaxTheme{
	110	+	KeywordType: "#00e5ff",
	111	+	NameBuiltin: "#00e5ff",
	112	+	}
	113	+	rows := th.HighlightCode("func splitFrontmatter(raw string) (string, string) {\n", "go")
	114	+	joined := strings.Join(rows, "\n")
	115	+	if strings.Contains(joined, "\x1b[38;5m") \|\| strings.Contains(joined, "\x1b[48;5m") {
	116	+	t.Fatalf("highlighting emitted a malformed extended color: %q", joined)
	117	+	}
	118	+	if bad := incompleteExtendedColor(joined); bad != "" {
	119	+	t.Fatalf("highlighting emitted an incomplete extended color %q: %q", bad, joined)
	120	+	}
	121	+	// The type keyword keeps a full 256-color foreground.
	122	+	idx := strings.Index(joined, "string")
	123	+	if idx < 0 {
	124	+	t.Fatalf("type keyword missing from highlighted output: %q", joined)
	125	+	}
	126	+	prefix := joined[:idx]
	127	+	esc := prefix[strings.LastIndex(prefix, "\x1b["):]
	128	+	if !regexp.MustCompile(`^\x1b\[(?:[0-9]+;)*38;5;[0-9]+m$`).MatchString(esc) {
	129	+	t.Fatalf("type keyword lost its color, got %q in %q", esc, joined)
	130	+	}
	131	+	}
	132	+
	133	+	// Every syntax color in the built-in palettes must come out of the
	134	+	// highlighting path as a complete SGR sequence.
	135	+	func TestHighlightCodeAllPalettesWellFormed(t *testing.T) {
	136	+	samples := map[string]string{
	137	+	"go": "package main\n\n// note\nfunc main() { s := \"x\"; _ = len(s) }\n",
	138	+	"python": "def f(x: int) -> str:\n # note\n return str(x)\n",
	139	+	"markdown": "# Title\n\nSome emphasis and bold and `code`.\n",
	140	+	}
	141	+	for _, th := range []Theme{Dark, Light} {
	142	+	for lang, src := range samples {
	143	+	joined := strings.Join(th.HighlightCode(src, lang), "\n")
	144	+	if bad := incompleteExtendedColor(joined); bad != "" {
	145	+	t.Errorf("%s emitted an incomplete extended color %q: %q", lang, bad, joined)
	146	+	}
	147	+	}
	148	+	}
	149	+	}

@@ -0,0 +1,149 @@
	1	+	package tui
	2	+
	3	+	import (
	4	+	"regexp"
	5	+	"strings"
	6	+	"testing"
	7	+	)
	8	+
	9	+	// The stripper must treat "38;5;N" / "38;2;R;G;B" as one unit. Pulling the
	10	+	// color index out on its own (when it happens to look like an ANSI
	11	+	// background code, e.g. 45, 48, 49 or 101) leaves "\x1b[38;5m", whose leftover
	12	+	// "5" terminals read as blink, so the token loses its color and blinks.
	13	+	func TestStripANSIBackgroundsKeepsExtendedColors(t *testing.T) {
	14	+	cases := []struct{ name, in, want string }{
	15	+	{"256 fg in background range", "\x1b[38;5;45mstring\x1b[0m", "\x1b[38;5;45mstring\x1b[0m"},
	16	+	{"256 fg index 48", "\x1b[38;5;48mx\x1b[0m", "\x1b[38;5;48mx\x1b[0m"},
	17	+	{"256 fg index 49", "\x1b[38;5;49mx\x1b[0m", "\x1b[38;5;49mx\x1b[0m"},
	18	+	{"256 fg bright range", "\x1b[38;5;101mcomment\x1b[0m", "\x1b[38;5;101mcomment\x1b[0m"},
	19	+	{"256 fg index 107", "\x1b[38;5;107mx\x1b[0m", "\x1b[38;5;107mx\x1b[0m"},
	20	+	{"256 fg index 41", "\x1b[38;5;41mx\x1b[0m", "\x1b[38;5;41mx\x1b[0m"},
	21	+	{"bold plus 256 fg", "\x1b[1;38;5;45mstring\x1b[0m", "\x1b[1;38;5;45mstring\x1b[0m"},
	22	+	{"truecolor fg", "\x1b[38;2;0;229;255mstring\x1b[0m", "\x1b[38;2;0;229;255mstring\x1b[0m"},
	23	+	{"plain text", "hello", "hello"},
	24	+	{"reset", "\x1b[0m", "\x1b[0m"},
	25	+
	26	+	{"256 bg dropped", "\x1b[48;5;45mbg\x1b[0m", "bg\x1b[0m"},
	27	+	{"256 bg dropped, fg kept", "\x1b[38;5;45m\x1b[48;5;0mtext", "\x1b[38;5;45mtext"},
	28	+	{"truecolor bg dropped", "\x1b[48;2;9;0;21mbg\x1b[0m", "bg\x1b[0m"},
	29	+	{"default bg dropped", "\x1b[49mbg", "bg"},
	30	+	{"ansi bg 41 dropped", "\x1b[41mbg", "bg"},
	31	+	{"ansi bg 101 dropped", "\x1b[101mbg", "bg"},
	32	+	{"truncated fg dropped", "\x1b[38;5mstring", "string"},
	33	+	{"truncated bg dropped", "\x1b[48;5mstring", "string"},
	34	+	{"truncated fg inside row", "\x1b[1m\x1b[38;5;109mfunc\x1b[0m\x1b[38;5m(", "\x1b[1m\x1b[38;5;109mfunc\x1b[0m("},
	35	+	}
	36	+	for _, c := range cases {
	37	+	if got := stripANSIBackgrounds(c.in); got != c.want {
	38	+	t.Errorf("%s:\n in %q\n got %q\n want %q", c.name, c.in, got, c.want)
	39	+	}
	40	+	}
	41	+	}
	42	+
	43	+	// incompleteExtendedColor returns the first SGR sequence in s whose extended
	44	+	// color introducer ("38" or "48") is missing part of its parameter run, for
	45	+	// example "\x1b[38;5m", which terminals read as a color-less SGR 38 followed
	46	+	// by SGR 5 (blink). Validation here is independent of the production helper.
	47	+	func incompleteExtendedColor(s string) string {
	48	+	numeric := func(p string) bool {
	49	+	if p == "" {
	50	+	return false
	51	+	}
	52	+	for _, r := range p {
	53	+	if r < '0' \|\| r > '9' {
	54	+	return false
	55	+	}
	56	+	}
	57	+	return true
	58	+	}
	59	+	for i := 0; i < len(s); {
	60	+	if s[i] != 0x1b \|\| i+1 >= len(s) \|\| s[i+1] != '[' {
	61	+	i++
	62	+	continue
	63	+	}
	64	+	j := i + 2
	65	+	for j < len(s) && !(s[j] >= 0x40 && s[j] <= 0x7e) {
	66	+	j++
	67	+	}
	68	+	if j >= len(s) {
	69	+	return s[i:]
	70	+	}
	71	+	if s[j] == 'm' {
	72	+	params := strings.Split(s[i+2:j], ";")
	73	+	for p := 0; p < len(params); p++ {
	74	+	if params[p] != "38" && params[p] != "48" {
	75	+	continue
	76	+	}
	77	+	need := 0
	78	+	switch {
	79	+	case p+1 < len(params) && params[p+1] == "5":
	80	+	need = 2
	81	+	case p+1 < len(params) && params[p+1] == "2":
	82	+	need = 4
	83	+	default:
	84	+	return s[i : j+1]
	85	+	}
	86	+	if p+need >= len(params) \|\| !numeric(params[p+2]) {
	87	+	return s[i : j+1]
	88	+	}
	89	+	if need == 4 {
	90	+	for k := p + 2; k <= p+4 && k < len(params); k++ {
	91	+	if !numeric(params[k]) {
	92	+	return s[i : j+1]
	93	+	}
	94	+	}
	95	+	}
	96	+	p += need
	97	+	}
	98	+	}
	99	+	i = j + 1
	100	+	}
	101	+	return ""
	102	+	}
	103	+
	104	+	// A theme color that chroma maps to xterm index 45 ("#00e5ff") must survive
	105	+	// highlighting with a renderable sequence: a complete "\x1b[38;5;45m", never
	106	+	// the malformed "\x1b[38;5m".
	107	+	func TestHighlightCodeExtendedColorIsWellFormed(t *testing.T) {
	108	+	th := Dark
	109	+	th.Syntax = SyntaxTheme{
	110	+	KeywordType: "#00e5ff",
	111	+	NameBuiltin: "#00e5ff",
	112	+	}
	113	+	rows := th.HighlightCode("func splitFrontmatter(raw string) (string, string) {\n", "go")
	114	+	joined := strings.Join(rows, "\n")
	115	+	if strings.Contains(joined, "\x1b[38;5m") \|\| strings.Contains(joined, "\x1b[48;5m") {
	116	+	t.Fatalf("highlighting emitted a malformed extended color: %q", joined)
	117	+	}
	118	+	if bad := incompleteExtendedColor(joined); bad != "" {
	119	+	t.Fatalf("highlighting emitted an incomplete extended color %q: %q", bad, joined)
	120	+	}
	121	+	// The type keyword keeps a full 256-color foreground.
	122	+	idx := strings.Index(joined, "string")
	123	+	if idx < 0 {
	124	+	t.Fatalf("type keyword missing from highlighted output: %q", joined)
	125	+	}
	126	+	prefix := joined[:idx]
	127	+	esc := prefix[strings.LastIndex(prefix, "\x1b["):]
	128	+	if !regexp.MustCompile(`^\x1b\[(?:[0-9]+;)*38;5;[0-9]+m$`).MatchString(esc) {
	129	+	t.Fatalf("type keyword lost its color, got %q in %q", esc, joined)
	130	+	}
	131	+	}
	132	+
	133	+	// Every syntax color in the built-in palettes must come out of the
	134	+	// highlighting path as a complete SGR sequence.
	135	+	func TestHighlightCodeAllPalettesWellFormed(t *testing.T) {
	136	+	samples := map[string]string{
	137	+	"go": "package main\n\n// note\nfunc main() { s := \"x\"; _ = len(s) }\n",
	138	+	"python": "def f(x: int) -> str:\n # note\n return str(x)\n",
	139	+	"markdown": "# Title\n\nSome emphasis and bold and `code`.\n",
	140	+	}
	141	+	for _, th := range []Theme{Dark, Light} {
	142	+	for lang, src := range samples {
	143	+	joined := strings.Join(th.HighlightCode(src, lang), "\n")
	144	+	if bad := incompleteExtendedColor(joined); bad != "" {
	145	+	t.Errorf("%s emitted an incomplete extended color %q: %q", lang, bad, joined)
	146	+	}
	147	+	}
	148	+	}
	149	+	}