1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package web
4
5import (
6 "bytes"
7 "context"
8 "fmt"
9 "html"
10 "html/template"
11 "path"
12 "runtime"
13 "strconv"
14 "strings"
15 "time"
16 "unicode"
17 "unicode/utf8"
18
19 "github.com/alecthomas/chroma/v2"
20 "github.com/alecthomas/chroma/v2/lexers"
21)
22
23// codeView is a source file ready for the code view: one <span class="line"> per
24// line, each with its number as an anchor, tokens as chroma CSS classes (the
25// colours are in static/css/56-chroma.css, no inline style).
26type codeView struct {
27 html template.HTML
28 lines int
29 // lang is the name of the language the file was highlighted as, "" for
30 // plain text.
31 lang string
32}
33
34// highlight renders src, the content of the file name, for the code view.
35// The language comes from the file name. Whatever goes wrong while lexing
36// (no lexer, a lexer error, panic or timeout, ctx done) degrades to plain
37// text: the source is never lost or altered. Characters that can mislead a
38// reader (bidi controls, invisible and control characters) are replaced by
39// visible markers.
40func highlight(ctx context.Context, name, src string) codeView {
41 src = normaliseText(src)
42 var lines [][]chroma.Token
43 lang := ""
44 if lx := lexerFor(name); lx != nil {
45 if toks, ok := tokenise(ctx, lx, src); ok {
46 lines, lang = chroma.SplitTokensIntoLines(toks), lx.Config().Name
47 }
48 }
49 if lines == nil {
50 for _, l := range splitSource(src) {
51 lines = append(lines, []chroma.Token{{Type: chroma.Text, Value: l}})
52 }
53 }
54 var b strings.Builder
55 b.Grow(len(src) + len(lines)*64)
56 for i, line := range lines {
57 n := strconv.Itoa(i + 1)
58 b.WriteString(`<span class="line" id="L` + n + `"><a class="ln" href="#L` + n + `">` + n + `</a>`)
59 for _, tk := range line {
60 v := strings.TrimRight(tk.Value, "\n")
61 if v == "" {
62 continue
63 }
64 if c := tokenClass(tk.Type); c != "" {
65 b.WriteString(`<span class="` + c + `">`)
66 writeMarked(&b, v)
67 b.WriteString(`</span>`)
68 } else {
69 writeMarked(&b, v)
70 }
71 }
72 b.WriteString("</span>\n")
73 }
74 // #nosec G203 -- every piece of the source goes through writeMarked, which
75 // escapes it; the rest is markup and class names of this file.
76 return codeView{html: template.HTML(b.String()), lines: len(lines), lang: lang}
77}
78
79// highlightBudget bounds the time spent lexing one file: a lexer on hostile
80// input can be slow, and the page then shows the file as plain text.
81const highlightBudget = 10 * time.Second
82
83// lexSlots bounds the lexers running at once: hostile files requested over
84// and over cannot take every CPU.
85var lexSlots = make(chan struct{}, runtime.GOMAXPROCS(0))
86
87// highlightCtx is highlight with a time budget and the cancellation of ctx:
88// the lexer stops at the next token once either is over, and the file is
89// shown as plain text. When every lexing slot is taken the file is shown as
90// plain text right away.
91func highlightCtx(ctx context.Context, name, src string) codeView {
92 select {
93 case lexSlots <- struct{}{}:
94 defer func() { <-lexSlots }()
95 default:
96 return highlightPlain(src)
97 }
98 ctx, cancel := context.WithTimeout(ctx, highlightBudget)
99 defer cancel()
100 return highlight(ctx, name, src)
101}
102
103// highlightPlain is the code view of src with no language.
104func highlightPlain(src string) codeView { return highlight(context.Background(), "", src) }
105
106// maxViewLines bounds the lines of the code view: every line costs about a
107// hundred bytes of HTML, and a megabyte of line feeds must not become a
108// hundred megabytes of page.
109const maxViewLines = 20000
110
111// countLines is the number of lines of src, as the code view numbers them.
112func countLines(src []byte) int {
113 n := bytes.Count(src, []byte{'\n'})
114 if len(src) > 0 && src[len(src)-1] != '\n' {
115 n++
116 }
117 return n
118}
119
120// normaliseText makes src valid UTF-8 with "\n" line ends.
121func normaliseText(src string) string {
122 if !utf8.ValidString(src) {
123 src = strings.ToValidUTF8(src, "\uFFFD")
124 }
125 if strings.Contains(src, "\r") {
126 src = strings.ReplaceAll(src, "\r\n", "\n")
127 src = strings.ReplaceAll(src, "\r", "\n")
128 }
129 return src
130}
131
132// lexerFor is the lexer for a file name, nil for plain text.
133func lexerFor(name string) chroma.Lexer {
134 lx := lexers.Match(path.Base(name))
135 if lx == nil {
136 return nil
137 }
138 return chroma.Coalesce(lx)
139}
140
141// tokenise lexes src, one token at a time so that it stops once ctx is done.
142// It reports false then, when the lexer panics (chroma lexers can, on odd
143// input), and when the tokens do not add up to the source, which is what a
144// lexer that gave up half-way (its regexp timeout) leaves behind. A lexer
145// may add a final line feed (EnsureNL), nothing else.
146func tokenise(ctx context.Context, lx chroma.Lexer, src string) (toks []chroma.Token, ok bool) {
147 return tokeniseInto(ctx, lx, src, nil)
148}
149
150// tokeniseInto is tokenise appending the tokens to buf (a buffer of length 0
151// that the caller reuses); the result is nil when it reports false.
152func tokeniseInto(ctx context.Context, lx chroma.Lexer, src string, buf []chroma.Token) (toks []chroma.Token, ok bool) {
153 defer func() {
154 if recover() != nil {
155 toks, ok = nil, false
156 }
157 }()
158 it, err := lx.Tokenise(&chroma.TokeniseOptions{State: "root", EnsureLF: true}, src)
159 if err != nil {
160 return nil, false
161 }
162 toks = buf
163 for t := it(); t != chroma.EOF; t = it() {
164 if len(toks)%256 == 0 && ctx.Err() != nil {
165 return nil, false
166 }
167 toks = append(toks, t)
168 }
169 if !tokensMatch(toks, src) {
170 return nil, false
171 }
172 return toks, true
173}
174
175// tokensMatch reports whether the values of the tokens, joined, are src but
176// for the line feeds that end either: it compares piece by piece, where
177// joining the tokens first would copy the whole file.
178func tokensMatch(toks []chroma.Token, src string) bool {
179 want := strings.TrimRight(src, "\n")
180 pos := 0
181 for _, t := range toks {
182 v := t.Value
183 rest := want[pos:]
184 if len(v) <= len(rest) {
185 if rest[:len(v)] != v {
186 return false
187 }
188 pos += len(v)
189 continue
190 }
191 // The token reaches past the end of the source: only line feeds may.
192 if v[:len(rest)] != rest || strings.Trim(v[len(rest):], "\n") != "" {
193 return false
194 }
195 pos = len(want)
196 }
197 return pos == len(want)
198}
199
200// splitSource splits plain text into lines, without the empty element after
201// a final line feed.
202func splitSource(src string) []string {
203 if src == "" {
204 return nil
205 }
206 ls := strings.Split(src, "\n")
207 if ls[len(ls)-1] == "" {
208 ls = ls[:len(ls)-1]
209 }
210 return ls
211}
212
213// tokenClass is the CSS class of a token type, "" for none, found the way
214// chroma's HTML formatter finds it: the type, then its parents.
215func tokenClass(t chroma.TokenType) string {
216 for t != 0 {
217 if c, ok := chroma.StandardTypes[t]; ok {
218 return c
219 }
220 t = t.Parent()
221 }
222 return chroma.StandardTypes[t]
223}
224
225// invisibleNames name the characters of the code view that are shown as
226// markers, for the tooltip. Any other invisible character is called by its
227// category.
228var invisibleNames = map[rune]string{
229 0x061C: "arabic letter mark",
230 0x200B: "zero width space",
231 0x200C: "zero width non-joiner",
232 0x200D: "zero width joiner",
233 0x200E: "left-to-right mark",
234 0x200F: "right-to-left mark",
235 0x202A: "left-to-right embedding",
236 0x202B: "right-to-left embedding",
237 0x202C: "pop directional formatting",
238 0x202D: "left-to-right override",
239 0x202E: "right-to-left override",
240 0x2028: "line separator",
241 0x2029: "paragraph separator",
242 0x2060: "word joiner",
243 0x2066: "left-to-right isolate",
244 0x2067: "right-to-left isolate",
245 0x2068: "first strong isolate",
246 0x2069: "pop directional isolate",
247 0xFEFF: "byte order mark",
248}
249
250// codePoint is "U+202E".
251func codePoint(r rune) string { return fmt.Sprintf("U+%04X", r) }
252
253// invisibleName is what the tooltip of a marker says.
254func invisibleName(r rune) string {
255 if n, ok := invisibleNames[r]; ok {
256 return n
257 }
258 if unicode.IsControl(r) {
259 return "control character"
260 }
261 return "invisible character"
262}
263
264// writeMarked writes s escaped for HTML, each invisible character replaced
265// by <span class="ctl" title="U+202E right-to-left override"><U+202E>
266// </span>. The character itself is never output.
267func writeMarked(b *strings.Builder, s string) {
268 for {
269 i := strings.IndexFunc(s, invisible)
270 if i < 0 {
271 b.WriteString(html.EscapeString(s))
272 return
273 }
274 b.WriteString(html.EscapeString(s[:i]))
275 r, size := utf8.DecodeRuneInString(s[i:])
276 cp := codePoint(r)
277 b.WriteString(`<span class="ctl" title="` + cp + ` ` + invisibleName(r) + `"><` + cp + `></span>`)
278 s = s[i+size:]
279 }
280}