1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package web
4
5import (
6 "context"
7 "fmt"
8 "html"
9 "html/template"
10 "path"
11 "runtime"
12 "strconv"
13 "strings"
14 "time"
15 "unicode"
16 "unicode/utf8"
17
18 "github.com/alecthomas/chroma/v2"
19 "github.com/alecthomas/chroma/v2/lexers"
20)
21
22// codeView is a source file ready for the code view: one <span class="line"> per
23// line, each with its number as an anchor, tokens as chroma CSS classes (the
24// colours are in static/css/56-chroma.css, no inline style).
25type codeView struct {
26 html template.HTML
27 lines int
28 // lang is the name of the language the file was highlighted as, "" for
29 // plain text.
30 lang string
31}
32
33// highlight renders src, the content of the file name, for the code view.
34// The language comes from the file name. Whatever goes wrong while lexing
35// (no lexer, a lexer error, panic or timeout, ctx done) degrades to plain
36// text: the source is never lost or altered. Characters that can mislead a
37// reader (bidi controls, invisible and control characters) are replaced by
38// visible markers.
39func highlight(ctx context.Context, name, src string) codeView {
40 src = normaliseText(src)
41 var lines [][]chroma.Token
42 lang := ""
43 if lx := lexerFor(name); lx != nil {
44 if toks, ok := tokenise(ctx, lx, src); ok {
45 lines, lang = chroma.SplitTokensIntoLines(toks), lx.Config().Name
46 }
47 }
48 if lines == nil {
49 for _, l := range splitSource(src) {
50 lines = append(lines, []chroma.Token{{Type: chroma.Text, Value: l}})
51 }
52 }
53 var b strings.Builder
54 b.Grow(len(src) + len(lines)*64)
55 for i, line := range lines {
56 n := strconv.Itoa(i + 1)
57 b.WriteString(`<span class="line" id="L` + n + `"><a class="ln" href="#L` + n + `">` + n + `</a>`)
58 for _, tk := range line {
59 v := strings.TrimRight(tk.Value, "\n")
60 if v == "" {
61 continue
62 }
63 if c := tokenClass(tk.Type); c != "" {
64 b.WriteString(`<span class="` + c + `">`)
65 writeMarked(&b, v)
66 b.WriteString(`</span>`)
67 } else {
68 writeMarked(&b, v)
69 }
70 }
71 b.WriteString("</span>\n")
72 }
73 // #nosec G203 -- every piece of the source goes through writeMarked, which
74 // escapes it; the rest is markup and class names of this file.
75 return codeView{html: template.HTML(b.String()), lines: len(lines), lang: lang}
76}
77
78// highlightBudget bounds the time spent lexing one file: a lexer on hostile
79// input can be slow, and the page then shows the file as plain text.
80const highlightBudget = 10 * time.Second
81
82// lexSlots bounds the lexers running at once: hostile files requested over
83// and over cannot take every CPU.
84var lexSlots = make(chan struct{}, runtime.GOMAXPROCS(0))
85
86// highlightCtx is highlight with a time budget and the cancellation of ctx:
87// the lexer stops at the next token once either is over, and the file is
88// shown as plain text. When every lexing slot is taken the file is shown as
89// plain text right away.
90func highlightCtx(ctx context.Context, name, src string) codeView {
91 select {
92 case lexSlots <- struct{}{}:
93 defer func() { <-lexSlots }()
94 default:
95 return highlightPlain(src)
96 }
97 ctx, cancel := context.WithTimeout(ctx, highlightBudget)
98 defer cancel()
99 return highlight(ctx, name, src)
100}
101
102// highlightPlain is the code view of src with no language.
103func highlightPlain(src string) codeView { return highlight(context.Background(), "", src) }
104
105// maxViewLines bounds the lines of the code view: every line costs about a
106// hundred bytes of HTML, and a megabyte of line feeds must not become a
107// hundred megabytes of page.
108const maxViewLines = 20000
109
110// countLines is the number of lines of src, as the code view numbers them.
111func countLines(src []byte) int {
112 n := strings.Count(string(src), "\n")
113 if len(src) > 0 && src[len(src)-1] != '\n' {
114 n++
115 }
116 return n
117}
118
119// normaliseText makes src valid UTF-8 with "\n" line ends.
120func normaliseText(src string) string {
121 if !utf8.ValidString(src) {
122 src = strings.ToValidUTF8(src, "\uFFFD")
123 }
124 if strings.Contains(src, "\r") {
125 src = strings.ReplaceAll(src, "\r\n", "\n")
126 src = strings.ReplaceAll(src, "\r", "\n")
127 }
128 return src
129}
130
131// lexerFor is the lexer for a file name, nil for plain text.
132func lexerFor(name string) chroma.Lexer {
133 lx := lexers.Match(path.Base(name))
134 if lx == nil {
135 return nil
136 }
137 return chroma.Coalesce(lx)
138}
139
140// tokenise lexes src, one token at a time so that it stops once ctx is done.
141// It reports false then, when the lexer panics (chroma lexers can, on odd
142// input), and when the tokens do not add up to the source, which is what a
143// lexer that gave up half-way (its regexp timeout) leaves behind. A lexer
144// may add a final line feed (EnsureNL), nothing else.
145func tokenise(ctx context.Context, lx chroma.Lexer, src string) (toks []chroma.Token, ok bool) {
146 defer func() {
147 if recover() != nil {
148 toks, ok = nil, false
149 }
150 }()
151 it, err := lx.Tokenise(&chroma.TokeniseOptions{State: "root", EnsureLF: true}, src)
152 if err != nil {
153 return nil, false
154 }
155 for t := it(); t != chroma.EOF; t = it() {
156 if len(toks)%256 == 0 && ctx.Err() != nil {
157 return nil, false
158 }
159 toks = append(toks, t)
160 }
161 if strings.TrimRight(joinTokens(toks), "\n") != strings.TrimRight(src, "\n") {
162 return nil, false
163 }
164 return toks, true
165}
166
167func joinTokens(toks []chroma.Token) string {
168 var b strings.Builder
169 for _, t := range toks {
170 b.WriteString(t.Value)
171 }
172 return b.String()
173}
174
175// splitSource splits plain text into lines, without the empty element after
176// a final line feed.
177func splitSource(src string) []string {
178 if src == "" {
179 return nil
180 }
181 ls := strings.Split(src, "\n")
182 if ls[len(ls)-1] == "" {
183 ls = ls[:len(ls)-1]
184 }
185 return ls
186}
187
188// tokenClass is the CSS class of a token type, "" for none, found the way
189// chroma's HTML formatter finds it: the type, then its parents.
190func tokenClass(t chroma.TokenType) string {
191 for t != 0 {
192 if c, ok := chroma.StandardTypes[t]; ok {
193 return c
194 }
195 t = t.Parent()
196 }
197 return chroma.StandardTypes[t]
198}
199
200// invisibleNames name the characters of the code view that are shown as
201// markers, for the tooltip. Any other invisible character is called by its
202// category.
203var invisibleNames = map[rune]string{
204 0x061C: "arabic letter mark",
205 0x200B: "zero width space",
206 0x200C: "zero width non-joiner",
207 0x200D: "zero width joiner",
208 0x200E: "left-to-right mark",
209 0x200F: "right-to-left mark",
210 0x202A: "left-to-right embedding",
211 0x202B: "right-to-left embedding",
212 0x202C: "pop directional formatting",
213 0x202D: "left-to-right override",
214 0x202E: "right-to-left override",
215 0x2028: "line separator",
216 0x2029: "paragraph separator",
217 0x2060: "word joiner",
218 0x2066: "left-to-right isolate",
219 0x2067: "right-to-left isolate",
220 0x2068: "first strong isolate",
221 0x2069: "pop directional isolate",
222 0xFEFF: "byte order mark",
223}
224
225// codePoint is "U+202E".
226func codePoint(r rune) string { return fmt.Sprintf("U+%04X", r) }
227
228// invisibleName is what the tooltip of a marker says.
229func invisibleName(r rune) string {
230 if n, ok := invisibleNames[r]; ok {
231 return n
232 }
233 if unicode.IsControl(r) {
234 return "control character"
235 }
236 return "invisible character"
237}
238
239// writeMarked writes s escaped for HTML, each invisible character replaced
240// by <span class="ctl" title="U+202E right-to-left override"><U+202E>
241// </span>. The character itself is never output.
242func writeMarked(b *strings.Builder, s string) {
243 for {
244 i := strings.IndexFunc(s, invisible)
245 if i < 0 {
246 b.WriteString(html.EscapeString(s))
247 return
248 }
249 b.WriteString(html.EscapeString(s[:i]))
250 r, size := utf8.DecodeRuneInString(s[i:])
251 cp := codePoint(r)
252 b.WriteString(`<span class="ctl" title="` + cp + ` ` + invisibleName(r) + `"><` + cp + `></span>`)
253 s = s[i+size:]
254 }
255}