1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package web
4
5import (
6 "bytes"
7 "context"
8 "fmt"
9 "html"
10 "html/template"
11 "path"
12 "runtime"
13 "strconv"
14 "strings"
15 "time"
16 "unicode"
17 "unicode/utf8"
18
19 "github.com/alecthomas/chroma/v2"
20 "github.com/alecthomas/chroma/v2/lexers"
21)
22
23// codeView is a source file ready for the code view: one <span class="line"> per
24// line, each with its number as an anchor, tokens as chroma CSS classes (the
25// colours are in static/css/56-chroma.css, no inline style).
26type codeView struct {
27 html template.HTML
28 lines int
29 // lang is the name of the language the file was highlighted as, "" for
30 // plain text.
31 lang string
32}
33
34// highlight renders src, the content of the file name, for the code view.
35// The language comes from the file name. Whatever goes wrong while lexing
36// (no lexer, a lexer error, panic or timeout, ctx done) degrades to plain
37// text: the source is never lost or altered. Characters that can mislead a
38// reader (bidi controls, invisible and control characters) are replaced by
39// visible markers.
40func highlight(ctx context.Context, name, src string) codeView {
41 src = normaliseText(src)
42 var lines [][]chroma.Token
43 lang := ""
44 if lx := lexerFor(name); lx != nil {
45 if toks, ok := tokenise(ctx, lx, src); ok {
46 lines, lang = chroma.SplitTokensIntoLines(toks), lx.Config().Name
47 }
48 }
49 if lines == nil {
50 for _, l := range splitSource(src) {
51 lines = append(lines, []chroma.Token{{Type: chroma.Text, Value: l}})
52 }
53 }
54 var b strings.Builder
55 b.Grow(len(src) + len(lines)*64)
56 for i, line := range lines {
57 n := strconv.Itoa(i + 1)
58 b.WriteString(`<span class="line" id="L` + n + `"><a class="ln" href="#L` + n + `">` + n + `</a>`)
59 for _, tk := range line {
60 v := strings.TrimRight(tk.Value, "\n")
61 if v == "" {
62 continue
63 }
64 if c := tokenClass(tk.Type); c != "" {
65 b.WriteString(`<span class="` + c + `">`)
66 writeMarked(&b, v)
67 b.WriteString(`</span>`)
68 } else {
69 writeMarked(&b, v)
70 }
71 }
72 b.WriteString("</span>\n")
73 }
74 // #nosec G203 -- every piece of the source goes through writeMarked, which
75 // escapes it; the rest is markup and class names of this file.
76 return codeView{html: template.HTML(b.String()), lines: len(lines), lang: lang}
77}
78
79// highlightBudget bounds the time spent lexing one file: a lexer on hostile
80// input can be slow, and the page then shows the file as plain text.
81const highlightBudget = 10 * time.Second
82
83// lexSlots bounds the lexers running at once: hostile files requested over
84// and over cannot take every CPU.
85var lexSlots = make(chan struct{}, runtime.GOMAXPROCS(0))
86
87// highlightCtx is highlight with a time budget and the cancellation of ctx:
88// the lexer stops at the next token once either is over, and the file is
89// shown as plain text. When every lexing slot is taken the file is shown as
90// plain text right away.
91func highlightCtx(ctx context.Context, name, src string) codeView {
92 select {
93 case lexSlots <- struct{}{}:
94 defer func() { <-lexSlots }()
95 default:
96 return highlightPlain(src)
97 }
98 ctx, cancel := context.WithTimeout(ctx, highlightBudget)
99 defer cancel()
100 return highlight(ctx, name, src)
101}
102
103// highlightPlain is the code view of src with no language.
104func highlightPlain(src string) codeView { return highlight(context.Background(), "", src) }
105
106// maxViewLines bounds the lines of the code view: every line costs about a
107// hundred bytes of HTML, and a megabyte of line feeds must not become a
108// hundred megabytes of page.
109const maxViewLines = 20000
110
111// countLines is the number of lines of src, as the code view numbers them.
112func countLines(src []byte) int {
113 n := bytes.Count(src, []byte{'\n'})
114 if len(src) > 0 && src[len(src)-1] != '\n' {
115 n++
116 }
117 return n
118}
119
120// normaliseText makes src valid UTF-8 with "\n" line ends.
121func normaliseText(src string) string {
122 if !utf8.ValidString(src) {
123 src = strings.ToValidUTF8(src, "\uFFFD")
124 }
125 if strings.Contains(src, "\r") {
126 src = strings.ReplaceAll(src, "\r\n", "\n")
127 src = strings.ReplaceAll(src, "\r", "\n")
128 }
129 return src
130}
131
132// lexerFor is the lexer for a file name, nil for plain text.
133func lexerFor(name string) chroma.Lexer {
134 lx := lexers.Match(path.Base(name))
135 if lx == nil {
136 return nil
137 }
138 return chroma.Coalesce(lx)
139}
140
141// tokenise lexes src, one token at a time so that it stops once ctx is done.
142// It reports false then, when the lexer panics (chroma lexers can, on odd
143// input), and when the tokens do not add up to the source, which is what a
144// lexer that gave up half-way (its regexp timeout) leaves behind. A lexer
145// may add a final line feed (EnsureNL), nothing else.
146func tokenise(ctx context.Context, lx chroma.Lexer, src string) (toks []chroma.Token, ok bool) {
147 defer func() {
148 if recover() != nil {
149 toks, ok = nil, false
150 }
151 }()
152 it, err := lx.Tokenise(&chroma.TokeniseOptions{State: "root", EnsureLF: true}, src)
153 if err != nil {
154 return nil, false
155 }
156 for t := it(); t != chroma.EOF; t = it() {
157 if len(toks)%256 == 0 && ctx.Err() != nil {
158 return nil, false
159 }
160 toks = append(toks, t)
161 }
162 if !tokensMatch(toks, src) {
163 return nil, false
164 }
165 return toks, true
166}
167
168// tokensMatch reports whether the values of the tokens, joined, are src but
169// for the line feeds that end either: it compares piece by piece, where
170// joining the tokens first would copy the whole file.
171func tokensMatch(toks []chroma.Token, src string) bool {
172 want := strings.TrimRight(src, "\n")
173 pos := 0
174 for _, t := range toks {
175 v := t.Value
176 rest := want[pos:]
177 if len(v) <= len(rest) {
178 if rest[:len(v)] != v {
179 return false
180 }
181 pos += len(v)
182 continue
183 }
184 // The token reaches past the end of the source: only line feeds may.
185 if v[:len(rest)] != rest || strings.Trim(v[len(rest):], "\n") != "" {
186 return false
187 }
188 pos = len(want)
189 }
190 return pos == len(want)
191}
192
193// splitSource splits plain text into lines, without the empty element after
194// a final line feed.
195func splitSource(src string) []string {
196 if src == "" {
197 return nil
198 }
199 ls := strings.Split(src, "\n")
200 if ls[len(ls)-1] == "" {
201 ls = ls[:len(ls)-1]
202 }
203 return ls
204}
205
206// tokenClass is the CSS class of a token type, "" for none, found the way
207// chroma's HTML formatter finds it: the type, then its parents.
208func tokenClass(t chroma.TokenType) string {
209 for t != 0 {
210 if c, ok := chroma.StandardTypes[t]; ok {
211 return c
212 }
213 t = t.Parent()
214 }
215 return chroma.StandardTypes[t]
216}
217
218// invisibleNames name the characters of the code view that are shown as
219// markers, for the tooltip. Any other invisible character is called by its
220// category.
221var invisibleNames = map[rune]string{
222 0x061C: "arabic letter mark",
223 0x200B: "zero width space",
224 0x200C: "zero width non-joiner",
225 0x200D: "zero width joiner",
226 0x200E: "left-to-right mark",
227 0x200F: "right-to-left mark",
228 0x202A: "left-to-right embedding",
229 0x202B: "right-to-left embedding",
230 0x202C: "pop directional formatting",
231 0x202D: "left-to-right override",
232 0x202E: "right-to-left override",
233 0x2028: "line separator",
234 0x2029: "paragraph separator",
235 0x2060: "word joiner",
236 0x2066: "left-to-right isolate",
237 0x2067: "right-to-left isolate",
238 0x2068: "first strong isolate",
239 0x2069: "pop directional isolate",
240 0xFEFF: "byte order mark",
241}
242
243// codePoint is "U+202E".
244func codePoint(r rune) string { return fmt.Sprintf("U+%04X", r) }
245
246// invisibleName is what the tooltip of a marker says.
247func invisibleName(r rune) string {
248 if n, ok := invisibleNames[r]; ok {
249 return n
250 }
251 if unicode.IsControl(r) {
252 return "control character"
253 }
254 return "invisible character"
255}
256
257// writeMarked writes s escaped for HTML, each invisible character replaced
258// by <span class="ctl" title="U+202E right-to-left override"><U+202E>
259// </span>. The character itself is never output.
260func writeMarked(b *strings.Builder, s string) {
261 for {
262 i := strings.IndexFunc(s, invisible)
263 if i < 0 {
264 b.WriteString(html.EscapeString(s))
265 return
266 }
267 b.WriteString(html.EscapeString(s[:i]))
268 r, size := utf8.DecodeRuneInString(s[i:])
269 cp := codePoint(r)
270 b.WriteString(`<span class="ctl" title="` + cp + ` ` + invisibleName(r) + `"><` + cp + `></span>`)
271 s = s[i+size:]
272 }
273}