Merge branch 'main' into feat/issue-mr-links
# Conflicts: # internal/adapters/web/markdown_test.go
4 files · +239 -6
M internal/adapters/web/highlight.go +7 -5
| @@ -166,7 +166,9 @@ func tokeniseInto(ctx context.Context, lx chroma.Lexer, src string, buf []chroma | |||
| 166 | 166 | } | |
| 167 | 167 | toks = append(toks, t) | |
| 168 | 168 | } | |
| 169 | - | if !tokensMatch(toks, src) { | |
| 169 | + | // The last tokens may have run past the deadline: the result is then | |
| 170 | + | // dropped, as promised. One iterator call itself cannot be preempted. | |
| 171 | + | if ctx.Err() != nil || !tokensMatch(toks, src) { | |
| 170 | 172 | return nil, false | |
| 171 | 173 | } | |
| 172 | 174 | return toks, true | |
| @@ -264,17 +266,17 @@ func invisibleName(r rune) string { | |||
| 264 | 266 | // writeMarked writes s escaped for HTML, each invisible character replaced | |
| 265 | 267 | // by <span class="ctl" title="U+202E right-to-left override"><U+202E> | |
| 266 | 268 | // </span>. The character itself is never output. | |
| 267 | - | func writeMarked(b *strings.Builder, s string) { | |
| 269 | + | func writeMarked(b interface{ WriteString(string) (int, error) }, s string) { | |
| 268 | 270 | for { | |
| 269 | 271 | i := strings.IndexFunc(s, invisible) | |
| 270 | 272 | if i < 0 { | |
| 271 | - | b.WriteString(html.EscapeString(s)) | |
| 273 | + | _, _ = b.WriteString(html.EscapeString(s)) | |
| 272 | 274 | return | |
| 273 | 275 | } | |
| 274 | - | b.WriteString(html.EscapeString(s[:i])) | |
| 276 | + | _, _ = b.WriteString(html.EscapeString(s[:i])) | |
| 275 | 277 | r, size := utf8.DecodeRuneInString(s[i:]) | |
| 276 | 278 | cp := codePoint(r) | |
| 277 | - | b.WriteString(`<span class="ctl" title="` + cp + ` ` + invisibleName(r) + `"><` + cp + `></span>`) | |
| 279 | + | _, _ = b.WriteString(`<span class="ctl" title="` + cp + ` ` + invisibleName(r) + `"><` + cp + `></span>`) | |
| 278 | 280 | s = s[i+size:] | |
| 279 | 281 | } | |
| 280 | 282 | } | |
| @@ -166,7 +166,9 @@ func tokeniseInto(ctx context.Context, lx chroma.Lexer, src string, buf []chroma | |||||
| 166 | } | 166 | } | ||
| 167 | toks = append(toks, t) | 167 | toks = append(toks, t) | ||
| 168 | } | 168 | } | ||
| 169 | - | if !tokensMatch(toks, src) { | 169 | + | // The last tokens may have run past the deadline: the result is then |
| 170 | + | // dropped, as promised. One iterator call itself cannot be preempted. | |||
| 171 | + | if ctx.Err() != nil || !tokensMatch(toks, src) { | |||
| 170 | return nil, false | 172 | return nil, false | ||
| 171 | } | 173 | } | ||
| 172 | return toks, true | 174 | return toks, true | ||
| @@ -264,17 +266,17 @@ func invisibleName(r rune) string { | |||||
| 264 | // writeMarked writes s escaped for HTML, each invisible character replaced | 266 | // writeMarked writes s escaped for HTML, each invisible character replaced | ||
| 265 | // by <span class="ctl" title="U+202E right-to-left override"><U+202E> | 267 | // by <span class="ctl" title="U+202E right-to-left override"><U+202E> | ||
| 266 | // </span>. The character itself is never output. | 268 | // </span>. The character itself is never output. | ||
| 267 | - | func writeMarked(b *strings.Builder, s string) { | 269 | + | func writeMarked(b interface{ WriteString(string) (int, error) }, s string) { |
| 268 | for { | 270 | for { | ||
| 269 | i := strings.IndexFunc(s, invisible) | 271 | i := strings.IndexFunc(s, invisible) | ||
| 270 | if i < 0 { | 272 | if i < 0 { | ||
| 271 | - | b.WriteString(html.EscapeString(s)) | 273 | + | _, _ = b.WriteString(html.EscapeString(s)) |
| 272 | return | 274 | return | ||
| 273 | } | 275 | } | ||
| 274 | - | b.WriteString(html.EscapeString(s[:i])) | 276 | + | _, _ = b.WriteString(html.EscapeString(s[:i])) |
| 275 | r, size := utf8.DecodeRuneInString(s[i:]) | 277 | r, size := utf8.DecodeRuneInString(s[i:]) | ||
| 276 | cp := codePoint(r) | 278 | cp := codePoint(r) | ||
| 277 | - | b.WriteString(`<span class="ctl" title="` + cp + ` ` + invisibleName(r) + `"><` + cp + `></span>`) | 279 | + | _, _ = b.WriteString(`<span class="ctl" title="` + cp + ` ` + invisibleName(r) + `"><` + cp + `></span>`) |
| 278 | s = s[i+size:] | 280 | s = s[i+size:] | ||
| 279 | } | 281 | } | ||
| 280 | } | 282 | } | ||
M internal/adapters/web/markdown.go +3 -1
| @@ -10,6 +10,7 @@ import ( | |||
| 10 | 10 | "regexp" | |
| 11 | 11 | "strconv" | |
| 12 | 12 | "strings" | |
| 13 | + | "time" | |
| 13 | 14 | ||
| 14 | 15 | "bnema.dev/ermite/internal/domain" | |
| 15 | 16 | "github.com/yuin/goldmark" | |
| @@ -46,7 +47,7 @@ func newMarkdown() *markdown { | |||
| 46 | 47 | goldmark.WithExtensions(extension.GFM), | |
| 47 | 48 | goldmark.WithParserOptions(parser.WithAutoHeadingID()), | |
| 48 | 49 | goldmark.WithRendererOptions( | |
| 49 | - | renderer.WithNodeRenderers(util.Prioritized(altRenderer{}, 100)), | |
| 50 | + | renderer.WithNodeRenderers(util.Prioritized(altRenderer{}, 100), util.Prioritized(fencedRenderer{}, 100)), | |
| 50 | 51 | // No html.WithUnsafe(): raw HTML blocks and inline tags are dropped. | |
| 51 | 52 | // Table alignment as an align attribute: the CSP forbids style="". | |
| 52 | 53 | extension.WithTableCellAlignMethod(extension.TableCellAlignAttribute), | |
| @@ -129,6 +130,7 @@ func (m *markdown) renderPolicy(source []byte, pol linkPolicy, res refResolver) | |||
| 129 | 130 | references(t, source, res) | |
| 130 | 131 | } | |
| 131 | 132 | ||
| 133 | + | doc.SetAttributeString(markdownBudgetKey, &markdownLexBudget{deadline: time.Now().Add(markdownLexTime)}) | |
| 132 | 134 | var buf bytes.Buffer | |
| 133 | 135 | if err := m.md.Renderer().Render(&buf, source, doc); err != nil { | |
| 134 | 136 | return "", nil, err | |
| @@ -10,6 +10,7 @@ import ( | |||||
| 10 | "regexp" | 10 | "regexp" | ||
| 11 | "strconv" | 11 | "strconv" | ||
| 12 | "strings" | 12 | "strings" | ||
| 13 | + | "time" | |||
| 13 | 14 | ||||
| 14 | "bnema.dev/ermite/internal/domain" | 15 | "bnema.dev/ermite/internal/domain" | ||
| 15 | "github.com/yuin/goldmark" | 16 | "github.com/yuin/goldmark" | ||
| @@ -46,7 +47,7 @@ func newMarkdown() *markdown { | |||||
| 46 | goldmark.WithExtensions(extension.GFM), | 47 | goldmark.WithExtensions(extension.GFM), | ||
| 47 | goldmark.WithParserOptions(parser.WithAutoHeadingID()), | 48 | goldmark.WithParserOptions(parser.WithAutoHeadingID()), | ||
| 48 | goldmark.WithRendererOptions( | 49 | goldmark.WithRendererOptions( | ||
| 49 | - | renderer.WithNodeRenderers(util.Prioritized(altRenderer{}, 100)), | 50 | + | renderer.WithNodeRenderers(util.Prioritized(altRenderer{}, 100), util.Prioritized(fencedRenderer{}, 100)), |
| 50 | // No html.WithUnsafe(): raw HTML blocks and inline tags are dropped. | 51 | // No html.WithUnsafe(): raw HTML blocks and inline tags are dropped. | ||
| 51 | // Table alignment as an align attribute: the CSP forbids style="". | 52 | // Table alignment as an align attribute: the CSP forbids style="". | ||
| 52 | extension.WithTableCellAlignMethod(extension.TableCellAlignAttribute), | 53 | extension.WithTableCellAlignMethod(extension.TableCellAlignAttribute), | ||
| @@ -129,6 +130,7 @@ func (m *markdown) renderPolicy(source []byte, pol linkPolicy, res refResolver) | |||||
| 129 | references(t, source, res) | 130 | references(t, source, res) | ||
| 130 | } | 131 | } | ||
| 131 | 132 | ||||
| 133 | + | doc.SetAttributeString(markdownBudgetKey, &markdownLexBudget{deadline: time.Now().Add(markdownLexTime)}) | |||
| 132 | var buf bytes.Buffer | 134 | var buf bytes.Buffer | ||
| 133 | if err := m.md.Renderer().Render(&buf, source, doc); err != nil { | 135 | if err := m.md.Renderer().Render(&buf, source, doc); err != nil { | ||
| 134 | return "", nil, err | 136 | return "", nil, err | ||
A internal/adapters/web/markdown_highlight.go +165 -0
| @@ -0,0 +1,165 @@ | |||
| 1 | + | // SPDX-License-Identifier: AGPL-3.0-or-later | |
| 2 | + | ||
| 3 | + | package web | |
| 4 | + | ||
| 5 | + | import ( | |
| 6 | + | "context" | |
| 7 | + | "strings" | |
| 8 | + | "time" | |
| 9 | + | ||
| 10 | + | "github.com/alecthomas/chroma/v2" | |
| 11 | + | "github.com/alecthomas/chroma/v2/lexers" | |
| 12 | + | "github.com/yuin/goldmark/ast" | |
| 13 | + | "github.com/yuin/goldmark/renderer" | |
| 14 | + | goldhtml "github.com/yuin/goldmark/renderer/html" | |
| 15 | + | "github.com/yuin/goldmark/util" | |
| 16 | + | ) | |
| 17 | + | ||
| 18 | + | const ( | |
| 19 | + | markdownBlockBytes = 64 << 10 | |
| 20 | + | markdownDocumentBytes = 256 << 10 | |
| 21 | + | markdownLexTime = 200 * time.Millisecond | |
| 22 | + | markdownBudgetKey = "markdown-lex-budget" | |
| 23 | + | ) | |
| 24 | + | ||
| 25 | + | // Budget belongs to the parsed document, never to the shared renderer. | |
| 26 | + | // Charge attempts, including failures, so repeated bad blocks cannot bypass it. | |
| 27 | + | type markdownLexBudget struct { | |
| 28 | + | bytes, blocks int | |
| 29 | + | deadline time.Time | |
| 30 | + | } | |
| 31 | + | ||
| 32 | + | type fencedRenderer struct{} | |
| 33 | + | ||
| 34 | + | // Capture goldmark's default renderer rather than duplicate its escaping rules. | |
| 35 | + | type fencedFallback struct{ render renderer.NodeRendererFunc } | |
| 36 | + | ||
| 37 | + | func (f *fencedFallback) Register(k ast.NodeKind, fn renderer.NodeRendererFunc) { | |
| 38 | + | if k == ast.KindFencedCodeBlock { | |
| 39 | + | f.render = fn | |
| 40 | + | } | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | func (fencedRenderer) RegisterFuncs(r renderer.NodeRendererFuncRegisterer) { | |
| 44 | + | var fallback fencedFallback | |
| 45 | + | goldhtml.NewRenderer().RegisterFuncs(&fallback) | |
| 46 | + | r.Register(ast.KindFencedCodeBlock, func(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { | |
| 47 | + | n := node.(*ast.FencedCodeBlock) | |
| 48 | + | if !entering { | |
| 49 | + | return ast.WalkContinue, nil | |
| 50 | + | } | |
| 51 | + | toks, lang, ok := markdownTokens(n, source) | |
| 52 | + | if !ok { | |
| 53 | + | // Default rendering is plain escaped text. Omit unsafe language classes. | |
| 54 | + | if !safeMarkdownLanguage(string(n.Language(source))) { | |
| 55 | + | plain := ast.NewFencedCodeBlock(nil) | |
| 56 | + | plain.SetLines(n.Lines()) | |
| 57 | + | n = plain | |
| 58 | + | } | |
| 59 | + | _, err := fallback.render(w, source, n, true) | |
| 60 | + | if err != nil { | |
| 61 | + | return ast.WalkStop, err | |
| 62 | + | } | |
| 63 | + | return fallback.render(w, source, n, false) | |
| 64 | + | } | |
| 65 | + | _, _ = w.WriteString(`<pre class="chroma"><code class="language-` + lang + `">`) | |
| 66 | + | // EnsureNL lexers may append a newline: never emit bytes not in the block. | |
| 67 | + | remaining := 0 | |
| 68 | + | for i := 0; i < n.Lines().Len(); i++ { | |
| 69 | + | line := n.Lines().At(i) | |
| 70 | + | remaining += len(line.Value(source)) | |
| 71 | + | } | |
| 72 | + | // The tokens are not coalesced (chroma's coalescer copies strings | |
| 73 | + | // quadratically): adjacent tokens of one class share a span here. | |
| 74 | + | open := "" | |
| 75 | + | for _, tk := range toks { | |
| 76 | + | v := tk.Value | |
| 77 | + | if len(v) > remaining { | |
| 78 | + | v = v[:remaining] | |
| 79 | + | } | |
| 80 | + | remaining -= len(v) | |
| 81 | + | if v == "" { | |
| 82 | + | continue | |
| 83 | + | } | |
| 84 | + | if c := tokenClass(tk.Type); c != open { | |
| 85 | + | if open != "" { | |
| 86 | + | _, _ = w.WriteString(`</span>`) | |
| 87 | + | } | |
| 88 | + | if c != "" { | |
| 89 | + | _, _ = w.WriteString(`<span class="` + c + `">`) | |
| 90 | + | } | |
| 91 | + | open = c | |
| 92 | + | } | |
| 93 | + | writeMarked(w, v) | |
| 94 | + | } | |
| 95 | + | if open != "" { | |
| 96 | + | _, _ = w.WriteString(`</span>`) | |
| 97 | + | } | |
| 98 | + | _, _ = w.WriteString("</code></pre>\n") | |
| 99 | + | return ast.WalkSkipChildren, nil | |
| 100 | + | }) | |
| 101 | + | } | |
| 102 | + | ||
| 103 | + | func safeMarkdownLanguage(s string) bool { | |
| 104 | + | if s == "" { | |
| 105 | + | return false | |
| 106 | + | } | |
| 107 | + | for _, c := range s { | |
| 108 | + | switch { | |
| 109 | + | case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', strings.ContainsRune("_+.#-", c): | |
| 110 | + | default: | |
| 111 | + | return false | |
| 112 | + | } | |
| 113 | + | } | |
| 114 | + | return true | |
| 115 | + | } | |
| 116 | + | ||
| 117 | + | func markdownTokens(n *ast.FencedCodeBlock, source []byte) ([]chroma.Token, string, bool) { | |
| 118 | + | var lang string | |
| 119 | + | if n.Info != nil { | |
| 120 | + | if words := strings.Fields(string(n.Info.Segment.Value(source))); len(words) > 0 { | |
| 121 | + | lang = words[0] | |
| 122 | + | } | |
| 123 | + | } | |
| 124 | + | if !safeMarkdownLanguage(lang) { | |
| 125 | + | return nil, "", false | |
| 126 | + | } | |
| 127 | + | value, _ := n.OwnerDocument().AttributeString(markdownBudgetKey) | |
| 128 | + | budget, ok := value.(*markdownLexBudget) | |
| 129 | + | if !ok || budget.blocks >= 128 || time.Now().After(budget.deadline) { | |
| 130 | + | return nil, "", false | |
| 131 | + | } | |
| 132 | + | size := 0 | |
| 133 | + | for i := 0; i < n.Lines().Len(); i++ { | |
| 134 | + | line := n.Lines().At(i) | |
| 135 | + | size += len(line.Value(source)) | |
| 136 | + | if size > markdownBlockBytes { | |
| 137 | + | return nil, "", false | |
| 138 | + | } | |
| 139 | + | } | |
| 140 | + | if size > markdownDocumentBytes-budget.bytes { | |
| 141 | + | return nil, "", false | |
| 142 | + | } | |
| 143 | + | budget.blocks++ | |
| 144 | + | budget.bytes += size | |
| 145 | + | lx := lexers.Get(lang) | |
| 146 | + | if lx == nil { | |
| 147 | + | return nil, "", false | |
| 148 | + | } | |
| 149 | + | select { | |
| 150 | + | case lexSlots <- struct{}{}: | |
| 151 | + | defer func() { <-lexSlots }() | |
| 152 | + | default: | |
| 153 | + | return nil, "", false | |
| 154 | + | } | |
| 155 | + | var code strings.Builder | |
| 156 | + | code.Grow(size) | |
| 157 | + | for i := 0; i < n.Lines().Len(); i++ { | |
| 158 | + | line := n.Lines().At(i) | |
| 159 | + | code.Write(line.Value(source)) | |
| 160 | + | } | |
| 161 | + | ctx, cancel := context.WithDeadline(context.Background(), budget.deadline) | |
| 162 | + | defer cancel() | |
| 163 | + | toks, ok := tokenise(ctx, lx, code.String()) | |
| 164 | + | return toks, lang, ok | |
| 165 | + | } | |
| @@ -0,0 +1,165 @@ | |||||
| 1 | + | // SPDX-License-Identifier: AGPL-3.0-or-later | |||
| 2 | + | ||||
| 3 | + | package web | |||
| 4 | + | ||||
| 5 | + | import ( | |||
| 6 | + | "context" | |||
| 7 | + | "strings" | |||
| 8 | + | "time" | |||
| 9 | + | ||||
| 10 | + | "github.com/alecthomas/chroma/v2" | |||
| 11 | + | "github.com/alecthomas/chroma/v2/lexers" | |||
| 12 | + | "github.com/yuin/goldmark/ast" | |||
| 13 | + | "github.com/yuin/goldmark/renderer" | |||
| 14 | + | goldhtml "github.com/yuin/goldmark/renderer/html" | |||
| 15 | + | "github.com/yuin/goldmark/util" | |||
| 16 | + | ) | |||
| 17 | + | ||||
| 18 | + | const ( | |||
| 19 | + | markdownBlockBytes = 64 << 10 | |||
| 20 | + | markdownDocumentBytes = 256 << 10 | |||
| 21 | + | markdownLexTime = 200 * time.Millisecond | |||
| 22 | + | markdownBudgetKey = "markdown-lex-budget" | |||
| 23 | + | ) | |||
| 24 | + | ||||
| 25 | + | // Budget belongs to the parsed document, never to the shared renderer. | |||
| 26 | + | // Charge attempts, including failures, so repeated bad blocks cannot bypass it. | |||
| 27 | + | type markdownLexBudget struct { | |||
| 28 | + | bytes, blocks int | |||
| 29 | + | deadline time.Time | |||
| 30 | + | } | |||
| 31 | + | ||||
| 32 | + | type fencedRenderer struct{} | |||
| 33 | + | ||||
| 34 | + | // Capture goldmark's default renderer rather than duplicate its escaping rules. | |||
| 35 | + | type fencedFallback struct{ render renderer.NodeRendererFunc } | |||
| 36 | + | ||||
| 37 | + | func (f *fencedFallback) Register(k ast.NodeKind, fn renderer.NodeRendererFunc) { | |||
| 38 | + | if k == ast.KindFencedCodeBlock { | |||
| 39 | + | f.render = fn | |||
| 40 | + | } | |||
| 41 | + | } | |||
| 42 | + | ||||
| 43 | + | func (fencedRenderer) RegisterFuncs(r renderer.NodeRendererFuncRegisterer) { | |||
| 44 | + | var fallback fencedFallback | |||
| 45 | + | goldhtml.NewRenderer().RegisterFuncs(&fallback) | |||
| 46 | + | r.Register(ast.KindFencedCodeBlock, func(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { | |||
| 47 | + | n := node.(*ast.FencedCodeBlock) | |||
| 48 | + | if !entering { | |||
| 49 | + | return ast.WalkContinue, nil | |||
| 50 | + | } | |||
| 51 | + | toks, lang, ok := markdownTokens(n, source) | |||
| 52 | + | if !ok { | |||
| 53 | + | // Default rendering is plain escaped text. Omit unsafe language classes. | |||
| 54 | + | if !safeMarkdownLanguage(string(n.Language(source))) { | |||
| 55 | + | plain := ast.NewFencedCodeBlock(nil) | |||
| 56 | + | plain.SetLines(n.Lines()) | |||
| 57 | + | n = plain | |||
| 58 | + | } | |||
| 59 | + | _, err := fallback.render(w, source, n, true) | |||
| 60 | + | if err != nil { | |||
| 61 | + | return ast.WalkStop, err | |||
| 62 | + | } | |||
| 63 | + | return fallback.render(w, source, n, false) | |||
| 64 | + | } | |||
| 65 | + | _, _ = w.WriteString(`<pre class="chroma"><code class="language-` + lang + `">`) | |||
| 66 | + | // EnsureNL lexers may append a newline: never emit bytes not in the block. | |||
| 67 | + | remaining := 0 | |||
| 68 | + | for i := 0; i < n.Lines().Len(); i++ { | |||
| 69 | + | line := n.Lines().At(i) | |||
| 70 | + | remaining += len(line.Value(source)) | |||
| 71 | + | } | |||
| 72 | + | // The tokens are not coalesced (chroma's coalescer copies strings | |||
| 73 | + | // quadratically): adjacent tokens of one class share a span here. | |||
| 74 | + | open := "" | |||
| 75 | + | for _, tk := range toks { | |||
| 76 | + | v := tk.Value | |||
| 77 | + | if len(v) > remaining { | |||
| 78 | + | v = v[:remaining] | |||
| 79 | + | } | |||
| 80 | + | remaining -= len(v) | |||
| 81 | + | if v == "" { | |||
| 82 | + | continue | |||
| 83 | + | } | |||
| 84 | + | if c := tokenClass(tk.Type); c != open { | |||
| 85 | + | if open != "" { | |||
| 86 | + | _, _ = w.WriteString(`</span>`) | |||
| 87 | + | } | |||
| 88 | + | if c != "" { | |||
| 89 | + | _, _ = w.WriteString(`<span class="` + c + `">`) | |||
| 90 | + | } | |||
| 91 | + | open = c | |||
| 92 | + | } | |||
| 93 | + | writeMarked(w, v) | |||
| 94 | + | } | |||
| 95 | + | if open != "" { | |||
| 96 | + | _, _ = w.WriteString(`</span>`) | |||
| 97 | + | } | |||
| 98 | + | _, _ = w.WriteString("</code></pre>\n") | |||
| 99 | + | return ast.WalkSkipChildren, nil | |||
| 100 | + | }) | |||
| 101 | + | } | |||
| 102 | + | ||||
| 103 | + | func safeMarkdownLanguage(s string) bool { | |||
| 104 | + | if s == "" { | |||
| 105 | + | return false | |||
| 106 | + | } | |||
| 107 | + | for _, c := range s { | |||
| 108 | + | switch { | |||
| 109 | + | case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', strings.ContainsRune("_+.#-", c): | |||
| 110 | + | default: | |||
| 111 | + | return false | |||
| 112 | + | } | |||
| 113 | + | } | |||
| 114 | + | return true | |||
| 115 | + | } | |||
| 116 | + | ||||
| 117 | + | func markdownTokens(n *ast.FencedCodeBlock, source []byte) ([]chroma.Token, string, bool) { | |||
| 118 | + | var lang string | |||
| 119 | + | if n.Info != nil { | |||
| 120 | + | if words := strings.Fields(string(n.Info.Segment.Value(source))); len(words) > 0 { | |||
| 121 | + | lang = words[0] | |||
| 122 | + | } | |||
| 123 | + | } | |||
| 124 | + | if !safeMarkdownLanguage(lang) { | |||
| 125 | + | return nil, "", false | |||
| 126 | + | } | |||
| 127 | + | value, _ := n.OwnerDocument().AttributeString(markdownBudgetKey) | |||
| 128 | + | budget, ok := value.(*markdownLexBudget) | |||
| 129 | + | if !ok || budget.blocks >= 128 || time.Now().After(budget.deadline) { | |||
| 130 | + | return nil, "", false | |||
| 131 | + | } | |||
| 132 | + | size := 0 | |||
| 133 | + | for i := 0; i < n.Lines().Len(); i++ { | |||
| 134 | + | line := n.Lines().At(i) | |||
| 135 | + | size += len(line.Value(source)) | |||
| 136 | + | if size > markdownBlockBytes { | |||
| 137 | + | return nil, "", false | |||
| 138 | + | } | |||
| 139 | + | } | |||
| 140 | + | if size > markdownDocumentBytes-budget.bytes { | |||
| 141 | + | return nil, "", false | |||
| 142 | + | } | |||
| 143 | + | budget.blocks++ | |||
| 144 | + | budget.bytes += size | |||
| 145 | + | lx := lexers.Get(lang) | |||
| 146 | + | if lx == nil { | |||
| 147 | + | return nil, "", false | |||
| 148 | + | } | |||
| 149 | + | select { | |||
| 150 | + | case lexSlots <- struct{}{}: | |||
| 151 | + | defer func() { <-lexSlots }() | |||
| 152 | + | default: | |||
| 153 | + | return nil, "", false | |||
| 154 | + | } | |||
| 155 | + | var code strings.Builder | |||
| 156 | + | code.Grow(size) | |||
| 157 | + | for i := 0; i < n.Lines().Len(); i++ { | |||
| 158 | + | line := n.Lines().At(i) | |||
| 159 | + | code.Write(line.Value(source)) | |||
| 160 | + | } | |||
| 161 | + | ctx, cancel := context.WithDeadline(context.Background(), budget.deadline) | |||
| 162 | + | defer cancel() | |||
| 163 | + | toks, ok := tokenise(ctx, lx, code.String()) | |||
| 164 | + | return toks, lang, ok | |||
| 165 | + | } | |||
M internal/adapters/web/markdown_test.go +64 -0
| @@ -319,3 +319,67 @@ func TestNumberReferenceTitleFromKindNotHref(t *testing.T) { | |||
| 319 | 319 | assert.Contains(t, string(h), `title="issue 3"`) | |
| 320 | 320 | assert.NotContains(t, string(h), "merge request 3") | |
| 321 | 321 | } | |
| 322 | + | ||
| 323 | + | func TestMarkdownFencedHighlight(t *testing.T) { | |
| 324 | + | tests := []struct { | |
| 325 | + | name, info, code string | |
| 326 | + | highlighted bool | |
| 327 | + | }{ | |
| 328 | + | {"go", "go", "package main\nfunc main() {}\n", true}, | |
| 329 | + | {"info suffix", "go example", "package main\n", true}, | |
| 330 | + | {"unknown", "not-a-real-language", "<script>\n", false}, | |
| 331 | + | {"empty", "", "<script>\n", false}, | |
| 332 | + | {"html", "html", "<script>alert(1)</script>\n", true}, | |
| 333 | + | {"injection", `go" onmouseover=x`, "package main\n", false}, | |
| 334 | + | {"oversize", "go", strings.Repeat("x", markdownBlockBytes+1) + "\n", false}, | |
| 335 | + | } | |
| 336 | + | for _, tt := range tests { | |
| 337 | + | t.Run(tt.name, func(t *testing.T) { | |
| 338 | + | src := "```" + tt.info + "\n" + tt.code + "```\n" | |
| 339 | + | for _, untrusted := range []bool{false, true} { | |
| 340 | + | var h string | |
| 341 | + | if untrusted { | |
| 342 | + | out, err := newMarkdown().renderUntrusted([]byte(src), "test-", nil) | |
| 343 | + | require.NoError(t, err) | |
| 344 | + | h = string(out) | |
| 345 | + | } else { | |
| 346 | + | h, _ = renderMD(t, src) | |
| 347 | + | } | |
| 348 | + | assert.Equal(t, tt.highlighted, strings.Contains(h, `<pre class="chroma">`)) | |
| 349 | + | assert.NotContains(t, h, "<script>") | |
| 350 | + | assert.NotContains(t, h, `style=`) | |
| 351 | + | assert.NotContains(t, h, `onmouseover`) | |
| 352 | + | if tt.name == "go" { | |
| 353 | + | assert.Contains(t, h, `<span class="kn">package</span>`) | |
| 354 | + | } | |
| 355 | + | if tt.name == "html" { | |
| 356 | + | assert.Contains(t, h, "<") | |
| 357 | + | assert.Contains(t, h, "script") | |
| 358 | + | } | |
| 359 | + | if !tt.highlighted { | |
| 360 | + | assert.NotContains(t, h, "<span") | |
| 361 | + | assert.Contains(t, h, "<pre><code") | |
| 362 | + | } | |
| 363 | + | } | |
| 364 | + | }) | |
| 365 | + | } | |
| 366 | + | } | |
| 367 | + | ||
| 368 | + | func TestMarkdownHighlightMarkers(t *testing.T) { | |
| 369 | + | h, _ := renderMD(t, "```go\n// invisible \u202e\n```\n") | |
| 370 | + | assert.Contains(t, h, `class="ctl"`) | |
| 371 | + | assert.Contains(t, h, "U+202E") | |
| 372 | + | assert.NotContains(t, h, "\u202e") | |
| 373 | + | } | |
| 374 | + | ||
| 375 | + | func TestMarkdownHighlightDocumentBudget(t *testing.T) { | |
| 376 | + | // Even thousands of tiny blocks may only attempt 128 lexes per document. | |
| 377 | + | h, _ := renderMD(t, strings.Repeat("```go\npackage main\n```\n\n", 1000)) | |
| 378 | + | assert.LessOrEqual(t, strings.Count(h, `<pre class="chroma">`), 128) | |
| 379 | + | assert.Contains(t, h, "<pre><code") | |
| 380 | + | // The byte budget also stops blocks before the block-count budget. | |
| 381 | + | block := "```go\n//" + strings.Repeat("x", markdownBlockBytes-4) + "\n```\n\n" | |
| 382 | + | h, _ = renderMD(t, strings.Repeat(block, 6)) | |
| 383 | + | assert.LessOrEqual(t, strings.Count(h, `<pre class="chroma">`), 4) | |
| 384 | + | assert.Contains(t, h, "<pre><code") | |
| 385 | + | } | |
| @@ -319,3 +319,67 @@ func TestNumberReferenceTitleFromKindNotHref(t *testing.T) { | |||||
| 319 | assert.Contains(t, string(h), `title="issue 3"`) | 319 | assert.Contains(t, string(h), `title="issue 3"`) | ||
| 320 | assert.NotContains(t, string(h), "merge request 3") | 320 | assert.NotContains(t, string(h), "merge request 3") | ||
| 321 | } | 321 | } | ||
| 322 | + | ||||
| 323 | + | func TestMarkdownFencedHighlight(t *testing.T) { | |||
| 324 | + | tests := []struct { | |||
| 325 | + | name, info, code string | |||
| 326 | + | highlighted bool | |||
| 327 | + | }{ | |||
| 328 | + | {"go", "go", "package main\nfunc main() {}\n", true}, | |||
| 329 | + | {"info suffix", "go example", "package main\n", true}, | |||
| 330 | + | {"unknown", "not-a-real-language", "<script>\n", false}, | |||
| 331 | + | {"empty", "", "<script>\n", false}, | |||
| 332 | + | {"html", "html", "<script>alert(1)</script>\n", true}, | |||
| 333 | + | {"injection", `go" onmouseover=x`, "package main\n", false}, | |||
| 334 | + | {"oversize", "go", strings.Repeat("x", markdownBlockBytes+1) + "\n", false}, | |||
| 335 | + | } | |||
| 336 | + | for _, tt := range tests { | |||
| 337 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 338 | + | src := "```" + tt.info + "\n" + tt.code + "```\n" | |||
| 339 | + | for _, untrusted := range []bool{false, true} { | |||
| 340 | + | var h string | |||
| 341 | + | if untrusted { | |||
| 342 | + | out, err := newMarkdown().renderUntrusted([]byte(src), "test-", nil) | |||
| 343 | + | require.NoError(t, err) | |||
| 344 | + | h = string(out) | |||
| 345 | + | } else { | |||
| 346 | + | h, _ = renderMD(t, src) | |||
| 347 | + | } | |||
| 348 | + | assert.Equal(t, tt.highlighted, strings.Contains(h, `<pre class="chroma">`)) | |||
| 349 | + | assert.NotContains(t, h, "<script>") | |||
| 350 | + | assert.NotContains(t, h, `style=`) | |||
| 351 | + | assert.NotContains(t, h, `onmouseover`) | |||
| 352 | + | if tt.name == "go" { | |||
| 353 | + | assert.Contains(t, h, `<span class="kn">package</span>`) | |||
| 354 | + | } | |||
| 355 | + | if tt.name == "html" { | |||
| 356 | + | assert.Contains(t, h, "<") | |||
| 357 | + | assert.Contains(t, h, "script") | |||
| 358 | + | } | |||
| 359 | + | if !tt.highlighted { | |||
| 360 | + | assert.NotContains(t, h, "<span") | |||
| 361 | + | assert.Contains(t, h, "<pre><code") | |||
| 362 | + | } | |||
| 363 | + | } | |||
| 364 | + | }) | |||
| 365 | + | } | |||
| 366 | + | } | |||
| 367 | + | ||||
| 368 | + | func TestMarkdownHighlightMarkers(t *testing.T) { | |||
| 369 | + | h, _ := renderMD(t, "```go\n// invisible \u202e\n```\n") | |||
| 370 | + | assert.Contains(t, h, `class="ctl"`) | |||
| 371 | + | assert.Contains(t, h, "U+202E") | |||
| 372 | + | assert.NotContains(t, h, "\u202e") | |||
| 373 | + | } | |||
| 374 | + | ||||
| 375 | + | func TestMarkdownHighlightDocumentBudget(t *testing.T) { | |||
| 376 | + | // Even thousands of tiny blocks may only attempt 128 lexes per document. | |||
| 377 | + | h, _ := renderMD(t, strings.Repeat("```go\npackage main\n```\n\n", 1000)) | |||
| 378 | + | assert.LessOrEqual(t, strings.Count(h, `<pre class="chroma">`), 128) | |||
| 379 | + | assert.Contains(t, h, "<pre><code") | |||
| 380 | + | // The byte budget also stops blocks before the block-count budget. | |||
| 381 | + | block := "```go\n//" + strings.Repeat("x", markdownBlockBytes-4) + "\n```\n\n" | |||
| 382 | + | h, _ = renderMD(t, strings.Repeat(block, 6)) | |||
| 383 | + | assert.LessOrEqual(t, strings.Count(h, `<pre class="chroma">`), 4) | |||
| 384 | + | assert.Contains(t, h, "<pre><code") | |||
| 385 | + | } | |||