fix(web): stop lexing at the highlight budget and survive lexer panics

commit: 52dd2efa719ae227443eb6bc282c62ed1f3107ef
author: brice · 11d ago · 2026-10-10 14:21 UTC
parent: 925a4ec
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

2 files · +60 -42

M internal/adapters/web/highlight.go +34 -31


@@ -32,16 +32,16 @@ type codeView struct {
32	32
33	33		// highlight renders src, the content of the file name, for the code view.
34	34		// The language comes from the file name. Whatever goes wrong while lexing
35		-	// (no lexer, a lexer error or timeout) degrades to plain text: the source is
36		-	// never lost or altered. Characters that can mislead a reader (bidi
37		-	// controls, invisible and control characters) are replaced by visible
38		-	// markers.
39		-	func highlight(name, src string) codeView {
	35	+	// (no lexer, a lexer error, panic or timeout, ctx done) degrades to plain
	36	+	// text: the source is never lost or altered. Characters that can mislead a
	37	+	// reader (bidi controls, invisible and control characters) are replaced by
	38	+	// visible markers.
	39	+	func highlight(ctx context.Context, name, src string) codeView {
40	40		src = normaliseText(src)
41	41		var lines [][]chroma.Token
42	42		lang := ""
43	43		if lx := lexerFor(name); lx != nil {
44		-	if toks, ok := tokenise(lx, src); ok {
	44	+	if toks, ok := tokenise(ctx, lx, src); ok {
45	45		lines, lang = chroma.SplitTokensIntoLines(toks), lx.Config().Name
46	46		}
47	47		}

@@ -79,37 +79,28 @@ func highlight(name, src string) codeView {
79	79		// input can be slow, and the page then shows the file as plain text.
80	80		const highlightBudget = 10 * time.Second
81	81
82		-	// lexSlots bounds the lexers running at once, those left to finish after
83		-	// their budget included: hostile files requested over and over cannot pile
84		-	// up CPU-bound goroutines.
	82	+	// lexSlots bounds the lexers running at once: hostile files requested over
	83	+	// and over cannot take every CPU.
85	84		var lexSlots = make(chan struct{}, runtime.GOMAXPROCS(0))
86	85
87		-	// highlightCtx is highlight with a time budget and the cancellation of ctx.
88		-	// A lexer cannot be interrupted, so one that overruns is left to finish on
89		-	// its own (its regexps have their own timeout) and its result is dropped.
90		-	// When every lexing slot is taken the file is shown as plain text.
	86	+	// highlightCtx is highlight with a time budget and the cancellation of ctx:
	87	+	// the lexer stops at the next token once either is over, and the file is
	88	+	// shown as plain text. When every lexing slot is taken the file is shown as
	89	+	// plain text right away.
91	90		func highlightCtx(ctx context.Context, name, src string) codeView {
92	91		select {
93	92		case lexSlots <- struct{}{}:
	93	+	defer func() { <-lexSlots }()
94	94		default:
95	95		return highlightPlain(src)
96	96		}
97		-	done := make(chan codeView, 1)
98		-	go func() {
99		-	defer func() { <-lexSlots }()
100		-	done <- highlight(name, src)
101		-	}()
102		-	select {
103		-	case v := <-done:
104		-	return v
105		-	case <-ctx.Done():
106		-	case <-time.After(highlightBudget):
107		-	}
108		-	return highlightPlain(src)
	97	+	ctx, cancel := context.WithTimeout(ctx, highlightBudget)
	98	+	defer cancel()
	99	+	return highlight(ctx, name, src)
109	100		}
110	101
111	102		// highlightPlain is the code view of src with no language.
112		-	func highlightPlain(src string) codeView { return highlight("", src) }
	103	+	func highlightPlain(src string) codeView { return highlight(context.Background(), "", src) }
113	104
114	105		// maxViewLines bounds the lines of the code view: every line costs about a
115	106		// hundred bytes of HTML, and a megabyte of line feeds must not become a

@@ -146,15 +137,27 @@ func lexerFor(name string) chroma.Lexer {
146	137		return chroma.Coalesce(lx)
147	138		}
148	139
149		-	// tokenise lexes src. It reports false when the tokens do not add up to the
150		-	// source, which is what a lexer that gave up half-way (its regexp timeout)
151		-	// leaves behind. A lexer may add a final line feed (EnsureNL), nothing else.
152		-	func tokenise(lx chroma.Lexer, src string) ([]chroma.Token, bool) {
	140	+	// tokenise lexes src, one token at a time so that it stops once ctx is done.
	141	+	// It reports false then, when the lexer panics (chroma lexers can, on odd
	142	+	// input), and when the tokens do not add up to the source, which is what a
	143	+	// lexer that gave up half-way (its regexp timeout) leaves behind. A lexer
	144	+	// may add a final line feed (EnsureNL), nothing else.
	145	+	func tokenise(ctx context.Context, lx chroma.Lexer, src string) (toks []chroma.Token, ok bool) {
	146	+	defer func() {
	147	+	if recover() != nil {
	148	+	toks, ok = nil, false
	149	+	}
	150	+	}()
153	151		it, err := lx.Tokenise(&chroma.TokeniseOptions{State: "root", EnsureLF: true}, src)
154	152		if err != nil {
155	153		return nil, false
156	154		}
157		-	toks := it.Tokens()
	155	+	for t := it(); t != chroma.EOF; t = it() {
	156	+	if len(toks)%256 == 0 && ctx.Err() != nil {
	157	+	return nil, false
	158	+	}
	159	+	toks = append(toks, t)
	160	+	}
158	161		if strings.TrimRight(joinTokens(toks), "\n") != strings.TrimRight(src, "\n") {
159	162		return nil, false
160	163		}

 // highlight renders src, the content of the file name, for the code view.
 // The language comes from the file name. Whatever goes wrong while lexing
-// (no lexer, a lexer error or timeout) degrades to plain text: the source is
+// (no lexer, a lexer error, panic or timeout, ctx done) degrades to plain
-// never lost or altered. Characters that can mislead a reader (bidi
+// text: the source is never lost or altered. Characters that can mislead a
-// controls, invisible and control characters) are replaced by visible
+// reader (bidi controls, invisible and control characters) are replaced by
-// markers.
+// visible markers.
-func highlight(name, src string) codeView {
+func highlight(ctx context.Context, name, src string) codeView {
 	src = normaliseText(src)
 	var lines [][]chroma.Token
 	lang := ""
 	if lx := lexerFor(name); lx != nil {
-		if toks, ok := tokenise(lx, src); ok {
+		if toks, ok := tokenise(ctx, lx, src); ok {
 			lines, lang = chroma.SplitTokensIntoLines(toks), lx.Config().Name
 		}
 	}
 // input can be slow, and the page then shows the file as plain text.
 const highlightBudget = 10 * time.Second
-// lexSlots bounds the lexers running at once, those left to finish after
+// lexSlots bounds the lexers running at once: hostile files requested over
-// their budget included: hostile files requested over and over cannot pile
+// and over cannot take every CPU.
-// up CPU-bound goroutines.
 var lexSlots = make(chan struct{}, runtime.GOMAXPROCS(0))
-// highlightCtx is highlight with a time budget and the cancellation of ctx.
+// highlightCtx is highlight with a time budget and the cancellation of ctx:
-// A lexer cannot be interrupted, so one that overruns is left to finish on
+// the lexer stops at the next token once either is over, and the file is
-// its own (its regexps have their own timeout) and its result is dropped.
+// shown as plain text. When every lexing slot is taken the file is shown as
-// When every lexing slot is taken the file is shown as plain text.
+// plain text right away.
 func highlightCtx(ctx context.Context, name, src string) codeView {
 	select {
 	case lexSlots <- struct{}{}:
+		defer func() { <-lexSlots }()
 	default:
 		return highlightPlain(src)
 	}
-	done := make(chan codeView, 1)
+	ctx, cancel := context.WithTimeout(ctx, highlightBudget)
-	go func() {
+	defer cancel()
-		defer func() { <-lexSlots }()
+	return highlight(ctx, name, src)
-		done <- highlight(name, src)
-	}()
-	select {
-	case v := <-done:
-		return v
-	case <-ctx.Done():
-	case <-time.After(highlightBudget):
-	}
-	return highlightPlain(src)
 }
 // highlightPlain is the code view of src with no language.
-func highlightPlain(src string) codeView { return highlight("", src) }
+func highlightPlain(src string) codeView { return highlight(context.Background(), "", src) }
 // maxViewLines bounds the lines of the code view: every line costs about a
 // hundred bytes of HTML, and a megabyte of line feeds must not become a
 	return chroma.Coalesce(lx)
 }
-// tokenise lexes src. It reports false when the tokens do not add up to the
+// tokenise lexes src, one token at a time so that it stops once ctx is done.
-// source, which is what a lexer that gave up half-way (its regexp timeout)
+// It reports false then, when the lexer panics (chroma lexers can, on odd
-// leaves behind. A lexer may add a final line feed (EnsureNL), nothing else.
+// input), and when the tokens do not add up to the source, which is what a
-func tokenise(lx chroma.Lexer, src string) ([]chroma.Token, bool) {
+// lexer that gave up half-way (its regexp timeout) leaves behind. A lexer
+// may add a final line feed (EnsureNL), nothing else.
+func tokenise(ctx context.Context, lx chroma.Lexer, src string) (toks []chroma.Token, ok bool) {
+	defer func() {
+		if recover() != nil {
+			toks, ok = nil, false
+		}
+	}()
 	it, err := lx.Tokenise(&chroma.TokeniseOptions{State: "root", EnsureLF: true}, src)
 	if err != nil {
 		return nil, false
 	}
-	toks := it.Tokens()
+	for t := it(); t != chroma.EOF; t = it() {
+		if len(toks)%256 == 0 && ctx.Err() != nil {
+			return nil, false
+		}
+		toks = append(toks, t)
+	}
 	if strings.TrimRight(joinTokens(toks), "\n") != strings.TrimRight(src, "\n") {
 		return nil, false
 	}

M internal/adapters/web/highlight_test.go +26 -11


@@ -7,6 +7,7 @@ import (
7	7		"strings"
8	8		"testing"
9	9
	10	+	"github.com/alecthomas/chroma/v2"
10	11		"github.com/stretchr/testify/assert"
11	12		"github.com/stretchr/testify/require"
12	13

@@ -14,7 +15,7 @@ import (
14	15		)
15	16
16	17		func TestHighlightLines(t *testing.T) {
17		-	v := highlight("a.go", "package main\n\nfunc main() {}\n")
	18	+	v := highlight(context.Background(), "a.go", "package main\n\nfunc main() {}\n")
18	19		assert.Equal(t, 3, v.lines)
19	20		assert.Equal(t, "Go", v.lang)
20	21		h := string(v.html)

@@ -23,20 +24,20 @@ func TestHighlightLines(t *testing.T) {
23	24		assert.NotContains(t, h, "style=")
24	25
25	26		// No final line feed, CRLF and lone CR are lines too.
26		-	assert.Equal(t, 2, highlight("x.txt", "a\nb").lines)
27		-	assert.Equal(t, 3, highlight("x.txt", "a\r\nb\rc").lines)
28		-	assert.Equal(t, 0, highlight("x.txt", "").lines)
	27	+	assert.Equal(t, 2, highlight(context.Background(), "x.txt", "a\nb").lines)
	28	+	assert.Equal(t, 3, highlight(context.Background(), "x.txt", "a\r\nb\rc").lines)
	29	+	assert.Equal(t, 0, highlight(context.Background(), "x.txt", "").lines)
29	30		}
30	31
31	32		func TestHighlightFallbackIsPlainAndEscaped(t *testing.T) {
32		-	v := highlight("unknown.zzz", "<script>alert(1)</script> & \"q\"\n")
	33	+	v := highlight(context.Background(), "unknown.zzz", "<script>alert(1)</script> & \"q\"\n")
33	34		assert.Equal(t, "", v.lang)
34	35		h := string(v.html)
35	36		assert.Contains(t, h, "<script>alert(1)</script> & "q"")
36	37		assert.NotContains(t, h, "<script>")
37	38
38	39		// Invalid UTF-8 is replaced, not passed on.
39		-	assert.Contains(t, string(highlight("x.txt", "a\xffb\n").html), "a\uFFFDb")
	40	+	assert.Contains(t, string(highlight(context.Background(), "x.txt", "a\xffb\n").html), "a\uFFFDb")
40	41		}
41	42
42	43		func TestHighlightMarksInvisibleCharacters(t *testing.T) {

@@ -44,17 +45,17 @@ func TestHighlightMarksInvisibleCharacters(t *testing.T) {
44	45		0x202E: "U+202E", 0x2066: "U+2066", 0x2069: "U+2069", 0x200B: "U+200B", 0xFEFF: "U+FEFF",
45	46		0x0000: "U+0000", 0x001B: "U+001B", 0x2028: "U+2028", 0x3164: "U+3164", 0x1F: "U+001F",
46	47		} {
47		-	h := string(highlight("x.go", "a"+string(r)+"b\n").html)
	48	+	h := string(highlight(context.Background(), "x.go", "a"+string(r)+"b\n").html)
48	49		assert.NotContains(t, h, string(r), cp)
49	50		assert.Contains(t, h, "<"+cp+"></span>", cp)
50	51		assert.Contains(t, h, `<span class="ctl" title="`+cp+" ", cp)
51	52		}
52	53		// A tab and ordinary non-ASCII text are left alone.
53		-	h := string(highlight("x.txt", "\tcafé 日本\n").html)
	54	+	h := string(highlight(context.Background(), "x.txt", "\tcafé 日本\n").html)
54	55		assert.Contains(t, h, "\tcafé 日本")
55	56		assert.NotContains(t, h, "ctl")
56	57		// Inside a token of a lexer too.
57		-	h = string(highlight("x.go", "x := \"a\u202eb\"\n").html)
	58	+	h = string(highlight(context.Background(), "x.go", "x := \"a\u202eb\"\n").html)
58	59		assert.NotContains(t, h, "\u202e")
59	60		assert.Contains(t, h, "<U+202E>")
60	61		}

@@ -76,7 +77,7 @@ func TestHighlightKeepsTheSource(t *testing.T) {
76	77		"Dockerfile": "FROM alpine\nRUN echo hi\n",
77	78		}
78	79		for name, src := range srcs {
79		-	v := highlight(name, src)
	80	+	v := highlight(context.Background(), name, src)
80	81		var text strings.Builder
81	82		for _, l := range strings.Split(string(v.html), "\n") {
82	83		l = strings.TrimSpace(l)

@@ -103,7 +104,21 @@ func TestHighlightCtxCancelled(t *testing.T) {
103	104		ctx, cancel := context.WithCancel(context.Background())
104	105		cancel()
105	106		v := highlightCtx(ctx, "a.go", "package a\n")
106		-	assert.Equal(t, 1, v.lines, "a cancelled context still gives the plain text, or the highlighted one")
	107	+	assert.Equal(t, 1, v.lines, "a cancelled context gives the plain text")
	108	+	assert.Empty(t, v.lang, "the lexer stopped")
	109	+	}
	110	+
	111	+	// panicLexer is a lexer that panics while lexing, as chroma lexers can.
	112	+	type panicLexer struct{ chroma.Lexer }
	113	+
	114	+	func (panicLexer) Tokenise(*chroma.TokeniseOptions, string) (chroma.Iterator, error) {
	115	+	return func() chroma.Token { panic("lexer bug") }, nil
	116	+	}
	117	+
	118	+	func TestTokeniseRecoversFromALexerPanic(t *testing.T) {
	119	+	toks, ok := tokenise(context.Background(), panicLexer{}, "x\n")
	120	+	assert.False(t, ok)
	121	+	assert.Nil(t, toks)
107	122		}
108	123
109	124		func TestSniffImage(t *testing.T) {

 	"strings"
 	"testing"
+	"github.com/alecthomas/chroma/v2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 func TestHighlightLines(t *testing.T) {
-	v := highlight("a.go", "package main\n\nfunc main() {}\n")
+	v := highlight(context.Background(), "a.go", "package main\n\nfunc main() {}\n")
 	assert.Equal(t, 3, v.lines)
 	assert.Equal(t, "Go", v.lang)
 	h := string(v.html)
 	assert.NotContains(t, h, "style=")
 	// No final line feed, CRLF and lone CR are lines too.
-	assert.Equal(t, 2, highlight("x.txt", "a\nb").lines)
+	assert.Equal(t, 2, highlight(context.Background(), "x.txt", "a\nb").lines)
-	assert.Equal(t, 3, highlight("x.txt", "a\r\nb\rc").lines)
+	assert.Equal(t, 3, highlight(context.Background(), "x.txt", "a\r\nb\rc").lines)
-	assert.Equal(t, 0, highlight("x.txt", "").lines)
+	assert.Equal(t, 0, highlight(context.Background(), "x.txt", "").lines)
 }
 func TestHighlightFallbackIsPlainAndEscaped(t *testing.T) {
-	v := highlight("unknown.zzz", "<script>alert(1)</script> & \"q\"\n")
+	v := highlight(context.Background(), "unknown.zzz", "<script>alert(1)</script> & \"q\"\n")
 	assert.Equal(t, "", v.lang)
 	h := string(v.html)
 	assert.Contains(t, h, "&lt;script&gt;alert(1)&lt;/script&gt; &amp; &#34;q&#34;")
 	assert.NotContains(t, h, "<script>")
 	// Invalid UTF-8 is replaced, not passed on.
-	assert.Contains(t, string(highlight("x.txt", "a\xffb\n").html), "a\uFFFDb")
+	assert.Contains(t, string(highlight(context.Background(), "x.txt", "a\xffb\n").html), "a\uFFFDb")
 }
 func TestHighlightMarksInvisibleCharacters(t *testing.T) {
 		0x202E: "U+202E", 0x2066: "U+2066", 0x2069: "U+2069", 0x200B: "U+200B", 0xFEFF: "U+FEFF",
 		0x0000: "U+0000", 0x001B: "U+001B", 0x2028: "U+2028", 0x3164: "U+3164", 0x1F: "U+001F",
 	} {
-		h := string(highlight("x.go", "a"+string(r)+"b\n").html)
+		h := string(highlight(context.Background(), "x.go", "a"+string(r)+"b\n").html)
 		assert.NotContains(t, h, string(r), cp)
 		assert.Contains(t, h, "&lt;"+cp+"&gt;</span>", cp)
 		assert.Contains(t, h, `<span class="ctl" title="`+cp+" ", cp)
 	}
 	// A tab and ordinary non-ASCII text are left alone.
-	h := string(highlight("x.txt", "\tcafé 日本\n").html)
+	h := string(highlight(context.Background(), "x.txt", "\tcafé 日本\n").html)
 	assert.Contains(t, h, "\tcafé 日本")
 	assert.NotContains(t, h, "ctl")
 	// Inside a token of a lexer too.
-	h = string(highlight("x.go", "x := \"a\u202eb\"\n").html)
+	h = string(highlight(context.Background(), "x.go", "x := \"a\u202eb\"\n").html)
 	assert.NotContains(t, h, "\u202e")
 	assert.Contains(t, h, "&lt;U+202E&gt;")
 }
 		"Dockerfile": "FROM alpine\nRUN echo hi\n",
 	}
 	for name, src := range srcs {
-		v := highlight(name, src)
+		v := highlight(context.Background(), name, src)
 		var text strings.Builder
 		for _, l := range strings.Split(string(v.html), "\n") {
 			l = strings.TrimSpace(l)
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 	v := highlightCtx(ctx, "a.go", "package a\n")
-	assert.Equal(t, 1, v.lines, "a cancelled context still gives the plain text, or the highlighted one")
+	assert.Equal(t, 1, v.lines, "a cancelled context gives the plain text")
+	assert.Empty(t, v.lang, "the lexer stopped")
+}
+// panicLexer is a lexer that panics while lexing, as chroma lexers can.
+type panicLexer struct{ chroma.Lexer }
+func (panicLexer) Tokenise(*chroma.TokeniseOptions, string) (chroma.Iterator, error) {
+	return func() chroma.Token { panic("lexer bug") }, nil
+}
+func TestTokeniseRecoversFromALexerPanic(t *testing.T) {
+	toks, ok := tokenise(context.Background(), panicLexer{}, "x\n")
+	assert.False(t, ok)
+	assert.Nil(t, toks)
 }
 func TestSniffImage(t *testing.T) {