1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package web
4
5import (
6 "context"
7 "slices"
8 "strconv"
9 "strings"
10 "sync"
11 "time"
12 "unicode/utf8"
13
14 "github.com/alecthomas/chroma/v2"
15
16 "bnema.dev/ermite/internal/domain"
17)
18
19// Syntax highlighting of the diffs. The tokens are the ones of the code view
20// (highlight.go, the same lexers, the same classes, the colours of
21// static/css/56-chroma.css); what differs is the input. A diff holds pieces
22// of files, not files, so each hunk is lexed on its own, as two sequences:
23// the old side (context and deleted lines) and the new side (context and
24// added lines). A multi-line construct that starts and ends inside a hunk
25// (a block comment, a raw string) is then coloured on all its lines, and a
26// deleted line gets the colours it had in the old file, not the ones the
27// added lines around it would give it. A hunk is lexed from its first line,
28// so a construct that opens above it is not known: the first lines may be
29// coloured as code (the usual limit of a diff view). The lines that /context
30// loads come from the whole file, which is lexed as a whole.
31//
32// The work is bounded, since a page is served under load and the lexers are
33// not cheap: a line, a file and a request have size limits (over them
34// the text stays plain), a request lexes with one of the lexing slots of the
35// code view (none free: plain) and a time budget, and the language is the
36// one of the file name (none: plain). Text that is not lexed, or whose tokens
37// do not add up to it, is shown as before.
38//
39// What a lexer finds is kept (hlcache.go) by the blobs the diff compares, so
40// that the visitors of one merge request or commit, and its live updates, lex
41// each file once. The colours are stored as runs (an end offset and a class a
42// piece of line), not as text: they are put back on the lines of the diff
43// that is being rendered.
44
45// maxDiffHLTotalBytes is the most text lexed for one request, the files of
46// the diff together: the files after it are plain. A variable for the tests.
47var maxDiffHLTotalBytes = 256 << 10
48
49const (
50 // maxDiffHLFileBytes is the most text of one file of a diff that is
51 // lexed. The diff limits (domain.DefaultDiffLimits) allow 2,000 changed lines a
52 // file; this keeps a file of very long lines out.
53 maxDiffHLFileBytes = 128 << 10
54 // maxDiffHLLine is the longest line that is lexed, in bytes: a minified
55 // file is the slowest input for a lexer and gains nothing from colour.
56 maxDiffHLLine = 2000
57 // diffHLBudget is the time a request spends lexing; once over, the rest
58 // of the diff is plain.
59 diffHLBudget = 2 * time.Second
60)
61
62// styledClasses are the token classes that 56-chroma.css gives a style. A
63// token of any other class has none worth sending: its text joins the plain
64// text around it. TestStyledClassesMatchCSS keeps it equal to the style sheet.
65var styledClasses = map[string]bool{
66 "c": true, "c1": true, "ch": true, "cm": true, "cs": true, "dl": true,
67 "k": true, "kc": true, "kd": true, "kn": true, "kr": true,
68 "nc": true, "ne": true, "nn": true, "ow": true,
69 "s": true, "s1": true, "s2": true, "sa": true, "sb": true, "sc": true, "sd": true,
70 "se": true, "sh": true, "si": true, "sr": true, "ss": true, "sx": true,
71}
72
73// diffClass is the CSS class of a token type, "" when it has no style.
74func diffClass(t chroma.TokenType) string {
75 if c := tokenClass(t); styledClasses[c] {
76 return c
77 }
78 return ""
79}
80
81// lexerMemoMax bounds the memo of lexers by file name: the names come from
82// the repositories, so it is emptied when full.
83const lexerMemoMax = 1024
84
85var lexerMemo struct {
86 sync.Mutex
87 m map[string]chroma.Lexer
88}
89
90// diffLexer is the lexer of a file name, nil for plain text. A name without
91// a language, or whose lexer only lexes plain text, has none. Finding a lexer
92// matches the name against every pattern chroma knows (it is as dear as lexing
93// a few hundred bytes), so the answer is kept by file name; lexers are safe to
94// share.
95func diffLexer(name string) chroma.Lexer {
96 lexerMemo.Lock()
97 lx, ok := lexerMemo.m[name]
98 lexerMemo.Unlock()
99 if ok {
100 return lx
101 }
102 if lx = lexerFor(name); lx != nil && lx.Config().Name == "plaintext" {
103 lx = nil
104 }
105 lexerMemo.Lock()
106 if lexerMemo.m == nil || len(lexerMemo.m) >= lexerMemoMax {
107 lexerMemo.m = map[string]chroma.Lexer{}
108 }
109 lexerMemo.m[name] = lx
110 lexerMemo.Unlock()
111 return lx
112}
113
114// lexingSlot takes one of the lexing slots of the code view without waiting;
115// release gives it back. ok is false when they are all taken.
116func lexingSlot() (release func(), ok bool) {
117 select {
118 case lexSlots <- struct{}{}:
119 return func() { <-lexSlots }, true
120 default:
121 return nil, false
122 }
123}
124
125// lexable reports whether lines can be lexed and shown with the colours put
126// back on them: valid UTF-8, no carriage return inside a line (the lexer
127// would take it for a line end and the lines would no longer line up), no
128// line too long.
129func lexable(lines []string) bool {
130 for _, l := range lines {
131 if len(l) > maxDiffHLLine || strings.ContainsRune(l, '\r') || !utf8.ValidString(l) {
132 return false
133 }
134 }
135 return true
136}
137
138// tokenBufs are the token buffers of lexLines.
139var tokenBufs = sync.Pool{New: func() any { b := make([]chroma.Token, 0, 4096); return &b }}
140
141// lexLines lexes lines, texts without their line end, as one sequence and
142// returns the tokens of each line. nil means no colours: the lexer failed,
143// ran out of time, or its tokens do not line up with the lines.
144func lexLines(ctx context.Context, lx chroma.Lexer, lines []string) [][]chroma.Token {
145 if len(lines) == 0 || !lexable(lines) {
146 return nil
147 }
148 n := 0
149 for _, l := range lines {
150 n += len(l) + 1
151 }
152 var b strings.Builder
153 b.Grow(n)
154 for _, l := range lines {
155 b.WriteString(l)
156 b.WriteByte('\n')
157 }
158 // The tokens of the whole text are a big slice, grown by doubling: one
159 // buffer is reused from a lexing to the next. The lines get copies of the
160 // tokens, so it is free again once they are split.
161 buf := tokenBufs.Get().(*[]chroma.Token)
162 toks, ok := tokeniseInto(ctx, lx, b.String(), (*buf)[:0])
163 if !ok {
164 tokenBufs.Put(buf)
165 return nil
166 }
167 per := chroma.SplitTokensIntoLines(toks)
168 clear(toks)
169 *buf = toks[:0]
170 tokenBufs.Put(buf)
171 if len(per) != len(lines) {
172 return nil
173 }
174 return per
175}
176
177// classList are the classes a run can have, "" (plain) first; a run holds the
178// index.
179var classList = func() []string {
180 l := []string{""}
181 for c := range styledClasses {
182 l = append(l, c)
183 }
184 slices.Sort(l[1:])
185 return l
186}()
187
188var classIndex = func() map[string]uint8 {
189 m := make(map[string]uint8, len(classList))
190 for i, c := range classList {
191 m[c] = uint8(i)
192 }
193 return m
194}()
195
196// hlRun is a piece of a line of one class: it ends at byte end.
197type hlRun struct {
198 end uint16
199 class uint8
200}
201
202// hlData are the colours of the lines of a file, the lines of its hunks in
203// order or all of them: the runs of line i are runs[lineEnd[i-1]:lineEnd[i]],
204// none for a plain line. It is read only once built, so it is shared by the
205// requests that use it.
206type hlData struct {
207 lineEnd []uint32
208 runs []hlRun
209}
210
211// size is what the data weighs, for the budget of the cache: what its
212// arrays hold, not what they use (computeFile trims them).
213func (d *hlData) size() int { return 64 + 4*cap(d.lineEnd) + 4*cap(d.runs) }
214
215// line is the runs of line i.
216func (d *hlData) line(i int) []hlRun {
217 lo := uint32(0)
218 if i > 0 {
219 lo = d.lineEnd[i-1]
220 }
221 return d.runs[lo:d.lineEnd[i]]
222}
223
224// segs puts the runs on the text of a line, in the room buf gives (nil: it
225// allocates); a line with invisible characters needs more and grows apart.
226// nil means plain: no run, or runs that do not cover the text (data of
227// another text, which no key of the cache lets through, but a wrong colour is
228// worse than none).
229func segs(text string, runs []hlRun, buf []Seg) []Seg {
230 if len(runs) == 0 || int(runs[len(runs)-1].end) != len(text) {
231 return nil
232 }
233 var out []Seg
234 if buf == nil {
235 out = make([]Seg, 0, len(runs))
236 } else {
237 out = buf[:0:len(runs)]
238 }
239 start := 0
240 for _, r := range runs {
241 end := int(r.end)
242 if end < start || end > len(text) || int(r.class) >= len(classList) {
243 return nil
244 }
245 out = appendSegs(out, text[start:end], classList[r.class])
246 start = end
247 }
248 return out
249}
250
251// hlBuilder builds the hlData of a file, line by line.
252type hlBuilder struct{ d hlData }
253
254// line adds the next line: its text and its tokens (nil for none). Tokens
255// that do not add up to the text leave the line plain.
256func (b *hlBuilder) line(text string, toks []chroma.Token) {
257 mark := len(b.d.runs)
258 if text != "" && len(toks) > 0 && !b.runs(text, toks) {
259 b.d.runs = b.d.runs[:mark]
260 }
261 b.d.lineEnd = append(b.d.lineEnd, uint32(len(b.d.runs)))
262}
263
264// runs appends the runs of a line, one for each stretch of tokens of the same
265// class; false if the tokens do not add up to the text.
266func (b *hlBuilder) runs(text string, toks []chroma.Token) bool {
267 pos, class := 0, ""
268 for _, t := range toks {
269 v := strings.TrimSuffix(t.Value, "\n")
270 if v == "" {
271 continue
272 }
273 if c := diffClass(t.Type); c != class {
274 if pos > 0 {
275 b.d.runs = append(b.d.runs, hlRun{end: uint16(pos), class: classIndex[class]})
276 }
277 class = c
278 }
279 pos += len(v)
280 if pos > len(text) {
281 return false
282 }
283 }
284 if pos != len(text) {
285 return false
286 }
287 b.d.runs = append(b.d.runs, hlRun{end: uint16(pos), class: classIndex[class]})
288 return true
289}
290
291// hunk adds the lines of a hunk, lexed as two sequences, one a side. Context
292// lines take their colours from the new side, or from the old side when the
293// hunk adds nothing: a side that no line needs is not lexed.
294func (b *hlBuilder) hunk(ctx context.Context, lx chroma.Lexer, src []domain.Line) {
295 var oldTexts, newTexts []string
296 hasAdd, hasDel := false, false
297 for _, l := range src {
298 t := strings.TrimSuffix(l.Text, "\r")
299 if l.Kind != domain.LineAddition {
300 oldTexts = append(oldTexts, t)
301 } else {
302 hasAdd = true
303 }
304 if l.Kind != domain.LineDeletion {
305 newTexts = append(newTexts, t)
306 } else {
307 hasDel = true
308 }
309 }
310 var oldToks, newToks [][]chroma.Token
311 if hasDel || !hasAdd {
312 oldToks = lexLines(ctx, lx, oldTexts)
313 }
314 if hasAdd {
315 newToks = lexLines(ctx, lx, newTexts)
316 }
317 tokens := func(all [][]chroma.Token, i int) []chroma.Token {
318 if i < len(all) {
319 return all[i]
320 }
321 return nil
322 }
323 oi, ni := 0, 0
324 for _, l := range src {
325 switch l.Kind {
326 case domain.LineDeletion:
327 b.line(oldTexts[oi], tokens(oldToks, oi))
328 oi++
329 case domain.LineAddition:
330 b.line(newTexts[ni], tokens(newToks, ni))
331 ni++
332 default:
333 if hasAdd {
334 b.line(newTexts[ni], tokens(newToks, ni))
335 } else {
336 b.line(oldTexts[oi], tokens(oldToks, oi))
337 }
338 oi++
339 ni++
340 }
341 }
342}
343
344// computeFile lexes the hunks of a file. ok is false when ctx ended, whatever
345// was lexed is then not the whole of it and is not kept.
346//
347//hexcheck:ignore suspicious-business-logic-in-adapter cuts the lines of a diff into coloured runs for the page; no rule is decided here
348func computeFile(ctx context.Context, lx chroma.Lexer, hunks []domain.Hunk) (d *hlData, ok bool) {
349 n := 0
350 for _, h := range hunks {
351 n += len(h.Lines)
352 }
353 b := hlBuilder{d: hlData{lineEnd: make([]uint32, 0, n), runs: make([]hlRun, 0, 4*n)}}
354 for _, h := range hunks {
355 if ctx.Err() != nil {
356 return nil, false
357 }
358 b.hunk(ctx, lx, h.Lines)
359 }
360 if ctx.Err() != nil {
361 return nil, false
362 }
363 // The runs were reserved for four a line; a cached entry keeps only what
364 // it uses (size counts the capacity).
365 if cap(b.d.runs) > len(b.d.runs)+len(b.d.runs)/4+16 {
366 b.d.runs = slices.Clone(b.d.runs)
367 }
368 return &b.d, true
369}
370
371// applyFile puts the colours on the lines of a file; false when the data is
372// not the one of this file (another number of lines), nothing is changed.
373func applyFile(d *hlData, f *DiffFile, hunks []domain.Hunk) bool {
374 n := 0
375 for _, h := range hunks {
376 n += len(h.Lines)
377 }
378 if len(d.lineEnd) != n || len(f.Hunks) != len(hunks) {
379 return false
380 }
381 i := 0
382 // The segments of all the lines come from one slab, not one allocation a
383 // line: that is most of what a cached page costs.
384 slab := make([]Seg, len(d.runs))
385 for j, h := range hunks {
386 if len(f.Hunks[j].Lines) != len(h.Lines) {
387 return false
388 }
389 for k, l := range h.Lines {
390 runs := d.line(i)
391 if s := segs(strings.TrimSuffix(l.Text, "\r"), runs, slab[:len(runs)]); s != nil {
392 f.Hunks[j].Lines[k].Segs = s
393 slab = slab[len(runs):]
394 }
395 i++
396 }
397 }
398 return true
399}
400
401// diffHLBytes is the size of the text of the lines of a file.
402func diffHLBytes(f domain.FileDiff) int {
403 n, lines := 0, 0
404 for _, h := range f.Hunks {
405 for _, l := range h.Lines {
406 n += len(l.Text) + 1
407 }
408 lines += len(h.Lines)
409 }
410 return n
411}
412
413// diffKey names the colours of a file of a diff: its blobs name the text of
414// both sides, so the hunks, and the language names the lexer. The number of
415// hunks and lines is a check on top. "" for a file with no blob ids, which is
416// not kept.
417func diffKey(lang string, f domain.FileDiff) string {
418 if f.OldID.IsZero() && f.NewID.IsZero() {
419 return ""
420 }
421 lines := 0
422 for _, h := range f.Hunks {
423 lines += len(h.Lines)
424 }
425 return "d\x00" + lang + "\x00" + f.OldID.String() + "\x00" + f.NewID.String() + "\x00" +
426 strconv.Itoa(len(f.Hunks)) + ":" + strconv.Itoa(lines)
427}
428
429// files is diffFiles with the syntax colours: the files of a diff for the
430// page, each one coloured from the cache or lexed. A nil cache lexes every
431// time.
432//
433// A file is left plain when it is binary, truncated, of no known language or
434// over a limit, and when it cannot be lexed now: every lexing slot is taken,
435// the request has used up its share of text or its time. A file that is in
436// the cache costs none of these.
437func (c *hlCache) files(ctx context.Context, files []domain.FileDiff) []DiffFile {
438 out := diffFiles(files)
439 deadline := time.Now().Add(diffHLBudget)
440 ctx, cancel := context.WithDeadline(ctx, deadline)
441 defer cancel()
442 // A lexing outlives the request that started it: the others that wait for
443 // it, and the cache, want the result. It ends at the same deadline.
444 bctx, bcancel := context.WithDeadline(context.WithoutCancel(ctx), deadline)
445 defer bcancel()
446
447 left := maxDiffHLTotalBytes
448 for i, f := range files {
449 if i >= len(out) || f.Binary || f.Omitted != domain.OmitNone || len(f.Hunks) == 0 || len(out[i].Hunks) != len(f.Hunks) {
450 continue
451 }
452 if ctx.Err() != nil {
453 return out
454 }
455 name := f.NewPath
456 if f.Status == domain.FileDeleted {
457 name = f.OldPath
458 }
459 // The size first: the language lookup is not free, and a file over a
460 // limit does not need it.
461 size := diffHLBytes(f)
462 if size > maxDiffHLFileBytes {
463 continue
464 }
465 // One lexer a file, set up once for all its lines and hunks.
466 lx := diffLexer(name)
467 if lx == nil {
468 continue
469 }
470 d := c.load(ctx, diffKey(lx.Config().Name, f), func() *hlData {
471 if size > left {
472 return nil
473 }
474 release, ok := lexingSlot()
475 if !ok {
476 return nil
477 }
478 defer release()
479 left -= size
480 d, _ := computeFile(bctx, lx, f.Hunks)
481 return d
482 })
483 if d != nil {
484 applyFile(d, &out[i], f.Hunks)
485 }
486 }
487 return out
488}
489
490// highlightContext colours the lines [from, to) of a file, the lines that
491// /context loads. The whole file is lexed (it is in memory already), not only
492// the lines shown: a comment or a string that opens above them, or closes
493// below them, is known. blob is the id of the file, what its colours are kept
494// under. A file over the limits is plain, as is nil.
495func (c *hlCache) highlightContext(ctx context.Context, blob, name string, lines []string, from, to int) [][]Seg {
496 if from >= to || to > len(lines) || len(lines) > maxViewLines {
497 return nil
498 }
499 lx := diffLexer(name)
500 if lx == nil {
501 return nil
502 }
503 size := 0
504 trimmed := make([]string, len(lines))
505 for i := range lines {
506 trimmed[i] = strings.TrimSuffix(lines[i], "\r")
507 size += len(trimmed[i]) + 1
508 }
509 if size > maxDiffHLFileBytes {
510 return nil
511 }
512 key := ""
513 if blob != "" {
514 key = "c\x00" + lx.Config().Name + "\x00" + blob + "\x00" + strconv.Itoa(len(lines))
515 }
516 deadline := time.Now().Add(diffHLBudget)
517 ctx, cancel := context.WithDeadline(ctx, deadline)
518 defer cancel()
519 bctx, bcancel := context.WithDeadline(context.WithoutCancel(ctx), deadline)
520 defer bcancel()
521 d := c.load(ctx, key, func() *hlData {
522 release, ok := lexingSlot()
523 if !ok {
524 return nil
525 }
526 defer release()
527 toks := lexLines(bctx, lx, trimmed)
528 if bctx.Err() != nil {
529 return nil
530 }
531 b := hlBuilder{d: hlData{lineEnd: make([]uint32, 0, len(lines)), runs: make([]hlRun, 0, 4*len(lines))}}
532 for i, t := range trimmed {
533 if toks != nil {
534 b.line(t, toks[i])
535 } else {
536 b.line(t, nil)
537 }
538 }
539 return &b.d
540 })
541 if d == nil || len(d.lineEnd) != len(lines) {
542 return nil
543 }
544 out := make([][]Seg, to-from)
545 for i := from; i < to; i++ {
546 out[i-from] = segs(trimmed[i], d.line(i), nil)
547 }
548 return out
549}