1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package web
4
5import (
6 "context"
7 "slices"
8 "strconv"
9 "strings"
10 "sync"
11 "time"
12 "unicode/utf8"
13
14 "github.com/alecthomas/chroma/v2"
15
16 "bnema.dev/ermite/internal/domain"
17)
18
19// Syntax highlighting of the diffs. The tokens are the ones of the code view
20// (highlight.go, the same lexers, the same classes, the colours of
21// static/css/56-chroma.css); what differs is the input. A diff holds pieces
22// of files, not files, so each hunk is lexed on its own, as two sequences:
23// the old side (context and deleted lines) and the new side (context and
24// added lines). A multi-line construct that starts and ends inside a hunk
25// (a block comment, a raw string) is then coloured on all its lines, and a
26// deleted line gets the colours it had in the old file, not the ones the
27// added lines around it would give it. A hunk is lexed from its first line,
28// so a construct that opens above it is not known: the first lines may be
29// coloured as code (the usual limit of a diff view). The lines that /context
30// loads come from the whole file, which is lexed as a whole.
31//
32// The work is bounded, since a page is served under load and the lexers are
33// not cheap: a line, a file and a request have size limits (over them
34// the text stays plain), a request lexes with one of the lexing slots of the
35// code view (none free: plain) and a time budget, and the language is the
36// one of the file name (none: plain). Text that is not lexed, or whose tokens
37// do not add up to it, is shown as before.
38//
39// What a lexer finds is kept (hlcache.go) by the blobs the diff compares, so
40// that the visitors of one merge request or commit, and its live updates, lex
41// each file once. The colours are stored as runs (an end offset and a class a
42// piece of line), not as text: they are put back on the lines of the diff
43// that is being rendered.
44
45// maxDiffHLTotalBytes is the most text lexed for one request, the files of
46// the diff together: the files after it are plain. A variable for the tests.
47var maxDiffHLTotalBytes = 256 << 10
48
49const (
50 // maxDiffHLFileBytes is the most text of one file of a diff that is
51 // lexed. The diff limits (domain.DefaultDiffLimits) allow 2,000 lines a
52 // file; this keeps a file of very long lines out.
53 maxDiffHLFileBytes = 128 << 10
54 // maxDiffHLLine is the longest line that is lexed, in bytes: a minified
55 // file is the slowest input for a lexer and gains nothing from colour.
56 maxDiffHLLine = 2000
57 // diffHLBudget is the time a request spends lexing; once over, the rest
58 // of the diff is plain.
59 diffHLBudget = 2 * time.Second
60)
61
62// styledClasses are the token classes that 56-chroma.css gives a style. A
63// token of any other class has none worth sending: its text joins the plain
64// text around it. TestStyledClassesMatchCSS keeps it equal to the style sheet.
65var styledClasses = map[string]bool{
66 "c": true, "c1": true, "ch": true, "cm": true, "cs": true, "dl": true,
67 "k": true, "kc": true, "kd": true, "kn": true, "kr": true,
68 "nc": true, "ne": true, "nn": true, "ow": true,
69 "s": true, "s1": true, "s2": true, "sa": true, "sb": true, "sc": true, "sd": true,
70 "se": true, "sh": true, "si": true, "sr": true, "ss": true, "sx": true,
71}
72
73// diffClass is the CSS class of a token type, "" when it has no style.
74func diffClass(t chroma.TokenType) string {
75 if c := tokenClass(t); styledClasses[c] {
76 return c
77 }
78 return ""
79}
80
81// lexerMemoMax bounds the memo of lexers by file name: the names come from
82// the repositories, so it is emptied when full.
83const lexerMemoMax = 1024
84
85var lexerMemo struct {
86 sync.Mutex
87 m map[string]chroma.Lexer
88}
89
90// diffLexer is the lexer of a file name, nil for plain text. A name without
91// a language, or whose lexer only lexes plain text, has none. Finding a lexer
92// matches the name against every pattern chroma knows (it is as dear as lexing
93// a few hundred bytes), so the answer is kept by file name; lexers are safe to
94// share.
95func diffLexer(name string) chroma.Lexer {
96 lexerMemo.Lock()
97 lx, ok := lexerMemo.m[name]
98 lexerMemo.Unlock()
99 if ok {
100 return lx
101 }
102 if lx = lexerFor(name); lx != nil && lx.Config().Name == "plaintext" {
103 lx = nil
104 }
105 lexerMemo.Lock()
106 if lexerMemo.m == nil || len(lexerMemo.m) >= lexerMemoMax {
107 lexerMemo.m = map[string]chroma.Lexer{}
108 }
109 lexerMemo.m[name] = lx
110 lexerMemo.Unlock()
111 return lx
112}
113
114// lexingSlot takes one of the lexing slots of the code view without waiting;
115// release gives it back. ok is false when they are all taken.
116func lexingSlot() (release func(), ok bool) {
117 select {
118 case lexSlots <- struct{}{}:
119 return func() { <-lexSlots }, true
120 default:
121 return nil, false
122 }
123}
124
125// lexable reports whether lines can be lexed and shown with the colours put
126// back on them: valid UTF-8, no carriage return inside a line (the lexer
127// would take it for a line end and the lines would no longer line up), no
128// line too long.
129func lexable(lines []string) bool {
130 for _, l := range lines {
131 if len(l) > maxDiffHLLine || strings.ContainsRune(l, '\r') || !utf8.ValidString(l) {
132 return false
133 }
134 }
135 return true
136}
137
138// tokenBufs are the token buffers of lexLines.
139var tokenBufs = sync.Pool{New: func() any { b := make([]chroma.Token, 0, 4096); return &b }}
140
141// lexLines lexes lines, texts without their line end, as one sequence and
142// returns the tokens of each line. nil means no colours: the lexer failed,
143// ran out of time, or its tokens do not line up with the lines.
144func lexLines(ctx context.Context, lx chroma.Lexer, lines []string) [][]chroma.Token {
145 if len(lines) == 0 || !lexable(lines) {
146 return nil
147 }
148 n := 0
149 for _, l := range lines {
150 n += len(l) + 1
151 }
152 var b strings.Builder
153 b.Grow(n)
154 for _, l := range lines {
155 b.WriteString(l)
156 b.WriteByte('\n')
157 }
158 // The tokens of the whole text are a big slice, grown by doubling: one
159 // buffer is reused from a lexing to the next. The lines get copies of the
160 // tokens, so it is free again once they are split.
161 buf := tokenBufs.Get().(*[]chroma.Token)
162 toks, ok := tokeniseInto(ctx, lx, b.String(), (*buf)[:0])
163 if !ok {
164 tokenBufs.Put(buf)
165 return nil
166 }
167 per := chroma.SplitTokensIntoLines(toks)
168 clear(toks)
169 *buf = toks[:0]
170 tokenBufs.Put(buf)
171 if len(per) != len(lines) {
172 return nil
173 }
174 return per
175}
176
177// classList are the classes a run can have, "" (plain) first; a run holds the
178// index.
179var classList = func() []string {
180 l := []string{""}
181 for c := range styledClasses {
182 l = append(l, c)
183 }
184 slices.Sort(l[1:])
185 return l
186}()
187
188var classIndex = func() map[string]uint8 {
189 m := make(map[string]uint8, len(classList))
190 for i, c := range classList {
191 m[c] = uint8(i)
192 }
193 return m
194}()
195
196// hlRun is a piece of a line of one class: it ends at byte end.
197type hlRun struct {
198 end uint16
199 class uint8
200}
201
202// hlData are the colours of the lines of a file, the lines of its hunks in
203// order or all of them: the runs of line i are runs[lineEnd[i-1]:lineEnd[i]],
204// none for a plain line. It is read only once built, so it is shared by the
205// requests that use it.
206type hlData struct {
207 lineEnd []uint32
208 runs []hlRun
209}
210
211// size is what the data weighs, for the budget of the cache: what its
212// arrays hold, not what they use (computeFile trims them).
213func (d *hlData) size() int { return 64 + 4*cap(d.lineEnd) + 4*cap(d.runs) }
214
215// line is the runs of line i.
216func (d *hlData) line(i int) []hlRun {
217 lo := uint32(0)
218 if i > 0 {
219 lo = d.lineEnd[i-1]
220 }
221 return d.runs[lo:d.lineEnd[i]]
222}
223
224// segs puts the runs on the text of a line, in the room buf gives (nil: it
225// allocates); a line with invisible characters needs more and grows apart.
226// nil means plain: no run, or runs that do not cover the text (data of
227// another text, which no key of the cache lets through, but a wrong colour is
228// worse than none).
229func segs(text string, runs []hlRun, buf []Seg) []Seg {
230 if len(runs) == 0 || int(runs[len(runs)-1].end) != len(text) {
231 return nil
232 }
233 var out []Seg
234 if buf == nil {
235 out = make([]Seg, 0, len(runs))
236 } else {
237 out = buf[:0:len(runs)]
238 }
239 start := 0
240 for _, r := range runs {
241 end := int(r.end)
242 if end < start || end > len(text) || int(r.class) >= len(classList) {
243 return nil
244 }
245 out = appendSegs(out, text[start:end], classList[r.class])
246 start = end
247 }
248 return out
249}
250
251// hlBuilder builds the hlData of a file, line by line.
252type hlBuilder struct{ d hlData }
253
254// line adds the next line: its text and its tokens (nil for none). Tokens
255// that do not add up to the text leave the line plain.
256func (b *hlBuilder) line(text string, toks []chroma.Token) {
257 mark := len(b.d.runs)
258 if text != "" && len(toks) > 0 && !b.runs(text, toks) {
259 b.d.runs = b.d.runs[:mark]
260 }
261 b.d.lineEnd = append(b.d.lineEnd, uint32(len(b.d.runs)))
262}
263
264// runs appends the runs of a line, one for each stretch of tokens of the same
265// class; false if the tokens do not add up to the text.
266func (b *hlBuilder) runs(text string, toks []chroma.Token) bool {
267 pos, class := 0, ""
268 for _, t := range toks {
269 v := strings.TrimSuffix(t.Value, "\n")
270 if v == "" {
271 continue
272 }
273 if c := diffClass(t.Type); c != class {
274 if pos > 0 {
275 b.d.runs = append(b.d.runs, hlRun{end: uint16(pos), class: classIndex[class]})
276 }
277 class = c
278 }
279 pos += len(v)
280 if pos > len(text) {
281 return false
282 }
283 }
284 if pos != len(text) {
285 return false
286 }
287 b.d.runs = append(b.d.runs, hlRun{end: uint16(pos), class: classIndex[class]})
288 return true
289}
290
291// hunk adds the lines of a hunk, lexed as two sequences, one a side. Context
292// lines take their colours from the new side, or from the old side when the
293// hunk adds nothing: a side that no line needs is not lexed.
294func (b *hlBuilder) hunk(ctx context.Context, lx chroma.Lexer, src []domain.Line) {
295 var oldTexts, newTexts []string
296 hasAdd, hasDel := false, false
297 for _, l := range src {
298 t := strings.TrimSuffix(l.Text, "\r")
299 if l.Kind != domain.LineAddition {
300 oldTexts = append(oldTexts, t)
301 } else {
302 hasAdd = true
303 }
304 if l.Kind != domain.LineDeletion {
305 newTexts = append(newTexts, t)
306 } else {
307 hasDel = true
308 }
309 }
310 var oldToks, newToks [][]chroma.Token
311 if hasDel || !hasAdd {
312 oldToks = lexLines(ctx, lx, oldTexts)
313 }
314 if hasAdd {
315 newToks = lexLines(ctx, lx, newTexts)
316 }
317 tokens := func(all [][]chroma.Token, i int) []chroma.Token {
318 if i < len(all) {
319 return all[i]
320 }
321 return nil
322 }
323 oi, ni := 0, 0
324 for _, l := range src {
325 switch l.Kind {
326 case domain.LineDeletion:
327 b.line(oldTexts[oi], tokens(oldToks, oi))
328 oi++
329 case domain.LineAddition:
330 b.line(newTexts[ni], tokens(newToks, ni))
331 ni++
332 default:
333 if hasAdd {
334 b.line(newTexts[ni], tokens(newToks, ni))
335 } else {
336 b.line(oldTexts[oi], tokens(oldToks, oi))
337 }
338 oi++
339 ni++
340 }
341 }
342}
343
344// computeFile lexes the hunks of a file. ok is false when ctx ended, whatever
345// was lexed is then not the whole of it and is not kept.
346//
347//hexcheck:ignore suspicious-business-logic-in-adapter cuts the lines of a diff into coloured runs for the page; no rule is decided here
348func computeFile(ctx context.Context, lx chroma.Lexer, hunks []domain.Hunk) (d *hlData, ok bool) {
349 n := 0
350 for _, h := range hunks {
351 n += len(h.Lines)
352 }
353 b := hlBuilder{d: hlData{lineEnd: make([]uint32, 0, n), runs: make([]hlRun, 0, 4*n)}}
354 for _, h := range hunks {
355 if ctx.Err() != nil {
356 return nil, false
357 }
358 b.hunk(ctx, lx, h.Lines)
359 }
360 if ctx.Err() != nil {
361 return nil, false
362 }
363 // The runs were reserved for four a line; a cached entry keeps only what
364 // it uses (size counts the capacity).
365 if cap(b.d.runs) > len(b.d.runs)+len(b.d.runs)/4+16 {
366 b.d.runs = slices.Clone(b.d.runs)
367 }
368 return &b.d, true
369}
370
371// applyFile puts the colours on the lines of a file; false when the data is
372// not the one of this file (another number of lines), nothing is changed.
373//
374//hexcheck:ignore suspicious-business-logic-in-adapter puts coloured runs on the lines of a diff for the page; no rule is decided here
375func applyFile(d *hlData, f *DiffFile, hunks []domain.Hunk) bool {
376 n := 0
377 for _, h := range hunks {
378 n += len(h.Lines)
379 }
380 if len(d.lineEnd) != n || len(f.Hunks) != len(hunks) {
381 return false
382 }
383 i := 0
384 // The segments of all the lines come from one slab, not one allocation a
385 // line: that is most of what a cached page costs.
386 slab := make([]Seg, len(d.runs))
387 for j, h := range hunks {
388 if len(f.Hunks[j].Lines) != len(h.Lines) {
389 return false
390 }
391 for k, l := range h.Lines {
392 runs := d.line(i)
393 if s := segs(strings.TrimSuffix(l.Text, "\r"), runs, slab[:len(runs)]); s != nil {
394 f.Hunks[j].Lines[k].Segs = s
395 slab = slab[len(runs):]
396 }
397 i++
398 }
399 }
400 return true
401}
402
403// diffHLBytes is the size of the text of the lines of a file.
404func diffHLBytes(f domain.FileDiff) int {
405 n, lines := 0, 0
406 for _, h := range f.Hunks {
407 for _, l := range h.Lines {
408 n += len(l.Text) + 1
409 }
410 lines += len(h.Lines)
411 }
412 return n
413}
414
415// diffKey names the colours of a file of a diff: its blobs name the text of
416// both sides, so the hunks, and the language names the lexer. The number of
417// hunks and lines is a check on top. "" for a file with no blob ids, which is
418// not kept.
419func diffKey(lang string, f domain.FileDiff) string {
420 if f.OldID.IsZero() && f.NewID.IsZero() {
421 return ""
422 }
423 lines := 0
424 for _, h := range f.Hunks {
425 lines += len(h.Lines)
426 }
427 return "d\x00" + lang + "\x00" + f.OldID.String() + "\x00" + f.NewID.String() + "\x00" +
428 strconv.Itoa(len(f.Hunks)) + ":" + strconv.Itoa(lines)
429}
430
431// files is diffFiles with the syntax colours: the files of a diff for the
432// page, each one coloured from the cache or lexed. A nil cache lexes every
433// time.
434//
435// A file is left plain when it is binary, truncated, of no known language or
436// over a limit, and when it cannot be lexed now: every lexing slot is taken,
437// the request has used up its share of text or its time. A file that is in
438// the cache costs none of these.
439func (c *hlCache) files(ctx context.Context, files []domain.FileDiff) []DiffFile {
440 out := diffFiles(files)
441 deadline := time.Now().Add(diffHLBudget)
442 ctx, cancel := context.WithDeadline(ctx, deadline)
443 defer cancel()
444 // A lexing outlives the request that started it: the others that wait for
445 // it, and the cache, want the result. It ends at the same deadline.
446 bctx, bcancel := context.WithDeadline(context.WithoutCancel(ctx), deadline)
447 defer bcancel()
448
449 left := maxDiffHLTotalBytes
450 for i, f := range files {
451 if i >= len(out) || f.Binary || f.Truncated || len(f.Hunks) == 0 || len(out[i].Hunks) != len(f.Hunks) {
452 continue
453 }
454 if ctx.Err() != nil {
455 return out
456 }
457 name := f.NewPath
458 if f.Status == domain.FileDeleted {
459 name = f.OldPath
460 }
461 // The size first: the language lookup is not free, and a file over a
462 // limit does not need it.
463 size := diffHLBytes(f)
464 if size > maxDiffHLFileBytes {
465 continue
466 }
467 // One lexer a file, set up once for all its lines and hunks.
468 lx := diffLexer(name)
469 if lx == nil {
470 continue
471 }
472 d := c.load(ctx, diffKey(lx.Config().Name, f), func() *hlData {
473 if size > left {
474 return nil
475 }
476 release, ok := lexingSlot()
477 if !ok {
478 return nil
479 }
480 defer release()
481 left -= size
482 d, _ := computeFile(bctx, lx, f.Hunks)
483 return d
484 })
485 if d != nil {
486 applyFile(d, &out[i], f.Hunks)
487 }
488 }
489 return out
490}
491
492// highlightContext colours the lines [from, to) of a file, the lines that
493// /context loads. The whole file is lexed (it is in memory already), not only
494// the lines shown: a comment or a string that opens above them, or closes
495// below them, is known. blob is the id of the file, what its colours are kept
496// under. A file over the limits is plain, as is nil.
497func (c *hlCache) highlightContext(ctx context.Context, blob, name string, lines []string, from, to int) [][]Seg {
498 if from >= to || to > len(lines) || len(lines) > maxViewLines {
499 return nil
500 }
501 lx := diffLexer(name)
502 if lx == nil {
503 return nil
504 }
505 size := 0
506 trimmed := make([]string, len(lines))
507 for i := range lines {
508 trimmed[i] = strings.TrimSuffix(lines[i], "\r")
509 size += len(trimmed[i]) + 1
510 }
511 if size > maxDiffHLFileBytes {
512 return nil
513 }
514 key := ""
515 if blob != "" {
516 key = "c\x00" + lx.Config().Name + "\x00" + blob + "\x00" + strconv.Itoa(len(lines))
517 }
518 deadline := time.Now().Add(diffHLBudget)
519 ctx, cancel := context.WithDeadline(ctx, deadline)
520 defer cancel()
521 bctx, bcancel := context.WithDeadline(context.WithoutCancel(ctx), deadline)
522 defer bcancel()
523 d := c.load(ctx, key, func() *hlData {
524 release, ok := lexingSlot()
525 if !ok {
526 return nil
527 }
528 defer release()
529 toks := lexLines(bctx, lx, trimmed)
530 if bctx.Err() != nil {
531 return nil
532 }
533 b := hlBuilder{d: hlData{lineEnd: make([]uint32, 0, len(lines)), runs: make([]hlRun, 0, 4*len(lines))}}
534 for i, t := range trimmed {
535 if toks != nil {
536 b.line(t, toks[i])
537 } else {
538 b.line(t, nil)
539 }
540 }
541 return &b.d
542 })
543 if d == nil || len(d.lineEnd) != len(lines) {
544 return nil
545 }
546 out := make([][]Seg, to-from)
547 for i := from; i < to; i++ {
548 out[i-from] = segs(trimmed[i], d.line(i), nil)
549 }
550 return out
551}