1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package web
4
5import (
6 "bytes"
7 "html/template"
8 "net/url"
9 "path"
10 "regexp"
11 "strconv"
12 "strings"
13 "time"
14
15 "bnema.dev/ermite/internal/domain"
16 "bnema.dev/ermite/internal/mediapolicy"
17 "github.com/yuin/goldmark"
18 "github.com/yuin/goldmark/ast"
19 "github.com/yuin/goldmark/extension"
20 "github.com/yuin/goldmark/parser"
21 "github.com/yuin/goldmark/renderer"
22 "github.com/yuin/goldmark/text"
23 "github.com/yuin/goldmark/util"
24)
25
26// headingIDPrefix keeps the ids of README headings apart from the ids of the
27// page (main, keys, cmd-list…): a heading must not be able to take one.
28const headingIDPrefix = "md-"
29
30// markdown renders the Markdown of a repository. Its output is safe to put in
31// a page: raw HTML is dropped, and links and images follow the policy below.
32//
33// Link policy:
34// - http and https links are kept, with rel="nofollow noopener noreferrer";
35// - relative links and links to the root of the repository are rewritten to
36// the blob route of the ref being viewed, never above the repository root;
37// - #fragments point at the (prefixed) heading ids;
38// - everything else (javascript:, data:, vbscript:, file:, mailto:,
39// protocol-relative //host, empty targets…) loses its link: the text stays;
40// - video images (.mp4, .webm, .mov, .m4v, .ogv) become native players
41// from the raw route or an allowlisted HTTP(S) host; other images show alt text;
42// - standalone links and autolinks to allowlisted hosts become players only
43// for video extensions or GitHub /user-attachments/assets/<id> paths.
44type markdown struct {
45 md goldmark.Markdown
46 videoHosts []string
47}
48
49func newMarkdown(hosts ...string) *markdown {
50 return &markdown{videoHosts: mediapolicy.Hosts(hosts), md: goldmark.New(
51 goldmark.WithExtensions(extension.GFM),
52 goldmark.WithParserOptions(parser.WithAutoHeadingID()),
53 goldmark.WithRendererOptions(
54 renderer.WithNodeRenderers(util.Prioritized(altRenderer{}, 100), util.Prioritized(fencedRenderer{}, 100)),
55 // No html.WithUnsafe(): raw HTML blocks and inline tags are dropped.
56 // Table alignment as an align attribute: the CSP forbids style="".
57 extension.WithTableCellAlignMethod(extension.TableCellAlignAttribute),
58 ),
59 )}
60}
61
62// render turns source into HTML and the outline of its headings. repo and ref
63// are where relative links point; dir is the directory of the Markdown file
64// in the repository, "" for the root.
65func (m *markdown) render(source []byte, repo, ref, dir string) (template.HTML, []Heading, error) {
66 return m.renderWithIDs(source, repo, ref, dir, headingIDPrefix)
67}
68
69// renderWithIDs is render with the prefix of the heading ids and of the
70// #fragment links chosen, so that several documents on one page (the summaries
71// of a run) do not share ids.
72func (m *markdown) renderWithIDs(source []byte, repo, ref, dir, idPrefix string) (template.HTML, []Heading, error) {
73 return m.renderPolicy(source, linkPolicy{repo: repo, ref: ref, dir: dir, idPrefix: idPrefix}, refResolver{})
74}
75
76// renderNote uses the repository Markdown policy, then adds local note links
77// and links to issues and merge requests.
78func (m *markdown) renderNote(source []byte, repo, ref, idPrefix string, refs refResolver) (template.HTML, error) {
79 h, _, err := m.renderPolicy(source, linkPolicy{repo: repo, ref: ref, idPrefix: idPrefix}, refs)
80 return h, err
81}
82
83// renderIssue renders the trusted body of an issue: the policy of render, plus
84// links to issues and merge requests.
85func (m *markdown) renderIssue(source []byte, repo, ref, idPrefix string, numbers func(int64) (string, domain.NumberKind, bool)) (template.HTML, error) {
86 h, _, err := m.renderPolicy(source, linkPolicy{repo: repo, ref: ref, idPrefix: idPrefix}, refResolver{numbers: numbers})
87 return h, err
88}
89
90// refResolver tells which references of a text are links.
91type refResolver struct {
92 // notes tells whether ^N names a note of the page (nil: no ^N links).
93 notes func(int64) bool
94 // numbers gives the page of the issue or merge request #N, when it exists
95 // (nil: no #N links). The href is built by the server, never from the text.
96 numbers func(int64) (href string, kind domain.NumberKind, ok bool)
97}
98
99// renderUntrusted renders Markdown written by an anonymous visitor (an issue
100// approved from the public form). On top of the policy of render, which already
101// drops raw HTML, this renderer never loads an image or video:
102// - only http, https and mailto links are kept, every one with
103// rel="nofollow noopener noreferrer ugc"; relative links, #fragments and
104// every other scheme lose their link and keep the text;
105// - nothing points into the repository.
106func (m *markdown) renderUntrusted(source []byte, idPrefix string, numbers func(int64) (string, domain.NumberKind, bool)) (template.HTML, error) {
107 h, _, err := m.renderPolicy(source, linkPolicy{idPrefix: idPrefix, untrusted: true}, refResolver{numbers: numbers})
108 return h, err
109}
110
111func (m *markdown) renderPolicy(source []byte, pol linkPolicy, res refResolver) (template.HTML, []Heading, error) {
112 pc := parser.NewContext(parser.WithIDs(&prefixedIDs{prefix: pol.idPrefix, inner: parser.NewContext().IDs()}))
113 doc := m.md.Parser().Parse(text.NewReader(source), parser.WithContext(pc))
114 pol.source = source
115 pol.videoHosts = m.videoHosts
116 // Collect eligible text before the policy unwraps rejected links: their
117 // labels must not accidentally become references either.
118 var refs []*ast.Text
119 if res.notes != nil || res.numbers != nil {
120 _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
121 if !entering {
122 return ast.WalkContinue, nil
123 }
124 switch n := n.(type) {
125 case *ast.CodeSpan, *ast.Link, *ast.AutoLink, *ast.Image, *ast.CodeBlock, *ast.FencedCodeBlock:
126 return ast.WalkSkipChildren, nil
127 case *ast.Text:
128 refs = append(refs, n)
129 }
130 return ast.WalkContinue, nil
131 })
132 }
133 pol.apply(doc)
134 for _, t := range refs {
135 references(t, source, res)
136 }
137
138 doc.SetAttributeString(markdownBudgetKey, &markdownLexBudget{deadline: time.Now().Add(markdownLexTime)})
139 var buf bytes.Buffer
140 if err := m.md.Renderer().Render(&buf, source, doc); err != nil {
141 return "", nil, err
142 }
143 // #nosec G203 -- goldmark without WithUnsafe escapes text and drops raw
144 // HTML; linkPolicy has removed every URL it does not allow.
145 return template.HTML(buf.String()), outline(doc, source), nil
146}
147
148// prefixedIDs prefixes the heading ids that goldmark generates.
149type prefixedIDs struct {
150 prefix string
151 inner parser.IDs
152}
153
154func (p *prefixedIDs) Generate(value []byte, kind ast.NodeKind) []byte {
155 return append([]byte(p.prefix), p.inner.Generate(value, kind)...)
156}
157
158func (p *prefixedIDs) Put(value []byte) { p.inner.Put(value) }
159
160// outline lists the headings of a document in order.
161func outline(doc ast.Node, source []byte) []Heading {
162 var out []Heading
163 _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
164 h, ok := n.(*ast.Heading)
165 if !ok || !entering {
166 return ast.WalkContinue, nil
167 }
168 id, _ := h.AttributeString("id")
169 idb, _ := id.([]byte)
170 out = append(out, Heading{Level: h.Level, ID: string(idb), Text: plainText(h, source)})
171 return ast.WalkSkipChildren, nil
172 })
173 return out
174}
175
176// plainText is the text of a node without markup.
177func plainText(n ast.Node, source []byte) string {
178 var b strings.Builder
179 _ = ast.Walk(n, func(c ast.Node, entering bool) (ast.WalkStatus, error) {
180 if !entering {
181 return ast.WalkContinue, nil
182 }
183 switch t := c.(type) {
184 case *ast.Text:
185 b.Write(t.Value(source))
186 if t.SoftLineBreak() || t.HardLineBreak() {
187 b.WriteByte(' ')
188 }
189 case *ast.String:
190 b.Write(t.Value)
191 }
192 return ast.WalkContinue, nil
193 })
194 return strings.TrimSpace(b.String())
195}
196
197// linkPolicy applies the link and image policy to a parsed document.
198type linkPolicy struct {
199 repo, ref, dir string
200 source []byte
201 // idPrefix is the prefix of the heading ids that #fragment links point at.
202 idPrefix string
203 // untrusted is the policy of text from an anonymous visitor.
204 untrusted bool
205 videoHosts []string
206 rawVideo bool
207}
208
209const (
210 relExternal = "nofollow noopener noreferrer"
211 relUntrusted = "nofollow noopener noreferrer ugc"
212)
213
214func (p linkPolicy) rel() []byte {
215 if p.untrusted {
216 return []byte(relUntrusted)
217 }
218 return []byte(relExternal)
219}
220
221func (p linkPolicy) apply(doc ast.Node) {
222 // Collect first, change after: the walk must not see the tree move.
223 var links, images, autos []ast.Node
224 _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
225 if !entering {
226 return ast.WalkContinue, nil
227 }
228 switch n.(type) {
229 case *ast.Link:
230 links = append(links, n)
231 case *ast.Image:
232 images = append(images, n)
233 case *ast.AutoLink:
234 autos = append(autos, n)
235 }
236 return ast.WalkContinue, nil
237 })
238 for _, n := range images {
239 p.image(n.(*ast.Image))
240 }
241 for _, n := range links {
242 l := n.(*ast.Link)
243 if p.standaloneVideo(l, string(l.Destination), plainText(l, p.source)) {
244 continue
245 }
246 dest, kind := p.target(string(l.Destination))
247 if kind == dropped {
248 unwrap(l)
249 continue
250 }
251 l.Destination = []byte(dest)
252 l.Title = nil
253 if kind == external {
254 l.SetAttributeString("rel", p.rel())
255 }
256 }
257 for _, n := range autos {
258 a := n.(*ast.AutoLink)
259 if p.standaloneVideo(a, string(a.URL(p.source)), string(a.Label(p.source))) {
260 continue
261 }
262 if a.AutoLinkType == ast.AutoLinkURL && isWebURL(string(a.URL(p.source))) {
263 a.SetAttributeString("rel", p.rel())
264 continue
265 }
266 if p.untrusted && a.AutoLinkType == ast.AutoLinkEmail && validMailbox(string(a.Label(p.source))) {
267 a.SetAttributeString("rel", p.rel())
268 continue
269 }
270 // An email address or another scheme: the label as plain text.
271 a.Parent().ReplaceChild(a.Parent(), a, ast.NewString(append([]byte(nil), a.Label(p.source)...)))
272 }
273}
274
275// unwrap replaces a node by its children.
276func unwrap(n ast.Node) {
277 parent := n.Parent()
278 for c := n.FirstChild(); c != nil; c = n.FirstChild() {
279 parent.InsertBefore(parent, n, c)
280 }
281 parent.RemoveChild(parent, n)
282}
283
284// image replaces a video image with a player, otherwise its alt text.
285func (p linkPolicy) image(img *ast.Image) {
286 alt := plainText(img, p.source)
287 if alt == "" {
288 alt = "image"
289 }
290 if src, ok := p.videoTarget(string(img.Destination), true); ok {
291 img.Parent().ReplaceChild(img.Parent(), img, &videoNode{src: src, alt: alt})
292 return
293 }
294 img.Parent().ReplaceChild(img.Parent(), img, &altSpan{alt: alt})
295}
296
297type linkKind int
298
299const (
300 dropped linkKind = iota
301 external
302 internal
303)
304
305// target applies the link policy to a destination.
306func (p linkPolicy) target(dest string) (string, linkKind) {
307 dest = strings.TrimSpace(dest)
308 if dest == "" || strings.ContainsAny(dest, "\x00\r\n\t\\") {
309 return "", dropped
310 }
311 if p.untrusted {
312 return untrustedTarget(dest)
313 }
314 if strings.HasPrefix(dest, "#") {
315 frag := strings.ToLower(strings.TrimPrefix(dest, "#"))
316 if frag == "" || strings.ContainsAny(frag, " \"'<>") {
317 return "", dropped
318 }
319 return "#" + p.idPrefix + frag, internal
320 }
321 if strings.HasPrefix(dest, "//") {
322 return "", dropped
323 }
324 u, err := url.Parse(dest)
325 if err != nil {
326 return "", dropped
327 }
328 switch strings.ToLower(u.Scheme) {
329 case "http", "https":
330 if u.Host == "" {
331 return "", dropped
332 }
333 return u.String(), external
334 case "":
335 default:
336 return "", dropped
337 }
338 if u.Host != "" || u.Opaque != "" || u.Path == "" {
339 return "", dropped
340 }
341 // A path in the repository: from the directory of the file, or from the
342 // root when it starts with a slash. path.Clean on a rooted path never
343 // leaves the root, so ../../ cannot climb out of the repository.
344 var rel string
345 if strings.HasPrefix(u.Path, "/") {
346 rel = path.Clean(u.Path)
347 } else {
348 rel = path.Clean("/" + path.Join(p.dir, u.Path))
349 }
350 rel = strings.TrimPrefix(rel, "/")
351 href := repoHref(p.repo)
352 if rel != "" {
353 route := "blob"
354 if p.rawVideo {
355 route = "raw"
356 }
357 href = repoHref(p.repo, route, p.ref, rel)
358 }
359 if u.Fragment != "" {
360 href += "#" + url.PathEscape(u.Fragment)
361 }
362 return href, internal
363}
364
365// untrustedTarget keeps absolute http and https links with a host, and mailto
366// links to one plain address.
367func untrustedTarget(dest string) (string, linkKind) {
368 if strings.HasPrefix(dest, "//") {
369 return "", dropped
370 }
371 u, err := url.Parse(dest)
372 if err != nil {
373 return "", dropped
374 }
375 switch strings.ToLower(u.Scheme) {
376 case "http", "https":
377 if u.Host == "" || u.User != nil {
378 return "", dropped
379 }
380 return u.String(), external
381 case "mailto":
382 if u.Opaque == "" || u.RawQuery != "" || !validMailbox(u.Opaque) {
383 return "", dropped
384 }
385 return "mailto:" + u.Opaque, external
386 }
387 return "", dropped
388}
389
390// validMailbox tells whether s is a bare address: no space, quote, angle
391// bracket, percent sign, comma or second at sign.
392func validMailbox(s string) bool {
393 local, host, ok := strings.Cut(s, "@")
394 return ok && local != "" && host != "" && len(s) <= 254 &&
395 !strings.ContainsAny(local+host, " \t\r\n\"'<>%,;?#\\@")
396}
397
398func isWebURL(s string) bool {
399 u, err := url.Parse(s)
400 if err != nil || u.Host == "" {
401 return false
402 }
403 sc := strings.ToLower(u.Scheme)
404 return sc == "http" || sc == "https"
405}
406
407// kindAlt is the kind of altSpan.
408var kindAlt = ast.NewNodeKind("AltSpan")
409
410// altSpan stands for an image: its alt text, in a span the stylesheet styles.
411type altSpan struct {
412 ast.BaseInline
413 alt string
414}
415
416func (n *altSpan) Kind() ast.NodeKind { return kindAlt }
417
418func (n *altSpan) Dump(source []byte, level int) { ast.DumpHelper(n, source, level, nil, nil) }
419
420// altRenderer renders altSpan nodes.
421type altRenderer struct{}
422
423func (altRenderer) RegisterFuncs(r renderer.NodeRendererFuncRegisterer) {
424 r.Register(kindVideo, func(w util.BufWriter, _ []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
425 if entering {
426 v := n.(*videoNode)
427 _, _ = w.WriteString(`<video controls preload="metadata" src="`)
428 _, _ = w.Write(util.EscapeHTML([]byte(v.src)))
429 _, _ = w.WriteString(`" aria-label="`)
430 _, _ = w.Write(util.EscapeHTML([]byte(v.alt)))
431 _, _ = w.WriteString(`"></video>`)
432 }
433 return ast.WalkSkipChildren, nil
434 })
435 r.Register(kindAlt, func(w util.BufWriter, _ []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
436 if entering {
437 _, _ = w.WriteString(`<span class="alt">[`)
438 _, _ = w.Write(util.EscapeHTML([]byte(n.(*altSpan).alt)))
439 _, _ = w.WriteString(`]</span>`)
440 }
441 return ast.WalkContinue, nil
442 })
443}
444
445// referencePattern matches ^N (groups 1-2) and #N (groups 3-4). The preceding
446// character is not part of a word; #N also rejects & and # so that numeric
447// entities such as ' stay text.
448var referencePattern = regexp.MustCompile(`(^|[^0-9A-Za-z_^])\^([1-9][0-9]{0,5})\b|(^|[^0-9A-Za-z_&#])#([1-9][0-9]{0,9})\b`)
449
450// references changes only text segments; code and link labels never enter
451// this function. New links deliberately bypass heading-prefix rewriting.
452func references(t *ast.Text, source []byte, res refResolver) {
453 parent := t.Parent()
454 if parent == nil {
455 return
456 }
457 value := t.Value(source)
458 offset := 0
459 for _, match := range referencePattern.FindAllSubmatchIndex(value, -1) {
460 isNote := match[4] >= 0
461 var from, to int
462 if isNote {
463 from, to = match[4], match[5]
464 } else {
465 from, to = match[8], match[9]
466 }
467 if (isNote && res.notes == nil) || (!isNote && res.numbers == nil) {
468 continue
469 }
470 id := string(value[from:to])
471 n, err := strconv.ParseInt(id, 10, 64)
472 if err != nil {
473 continue
474 }
475 start := from - 1
476 // Markdown punctuation may split adjacent text into separate AST nodes.
477 // Check the original preceding byte as well as the segment-local regex.
478 if at := t.Segment.Start + start; at > 0 {
479 prev := source[at-1]
480 if prev >= '0' && prev <= '9' || prev >= 'A' && prev <= 'Z' || prev >= 'a' && prev <= 'z' || prev == '_' {
481 continue
482 }
483 if isNote && prev == '^' || !isNote && (prev == '&' || prev == '#') {
484 continue
485 }
486 }
487 link := ast.NewLink()
488 if isNote {
489 if !res.notes(n) {
490 continue
491 }
492 link.Destination = []byte("#c" + id)
493 link.Title = []byte("note " + id)
494 } else {
495 href, kind, ok := res.numbers(n)
496 if !ok {
497 continue
498 }
499 link.Destination = []byte(href)
500 title := "issue "
501 if kind == domain.NumberMergeRequest {
502 title = "merge request "
503 }
504 link.Title = []byte(title + id)
505 }
506 if start > offset {
507 parent.InsertBefore(parent, t, ast.NewTextSegment(text.NewSegment(t.Segment.Start+offset, t.Segment.Start+start)))
508 }
509 link.SetAttributeString("class", []byte("ref"))
510 link.AppendChild(link, ast.NewString(value[start:to]))
511 parent.InsertBefore(parent, t, link)
512 offset = to
513 }
514 if offset == 0 {
515 return
516 }
517 tail := ast.NewTextSegment(text.NewSegment(t.Segment.Start+offset, t.Segment.Stop))
518 tail.SetSoftLineBreak(t.SoftLineBreak())
519 tail.SetHardLineBreak(t.HardLineBreak())
520 parent.InsertBefore(parent, t, tail)
521 parent.RemoveChild(parent, t)
522}
523
524// videoTarget uses the ordinary destination policy before restricting media origins.
525func (p linkPolicy) videoTarget(dest string, requireExtension bool) (string, bool) {
526 if p.untrusted {
527 return "", false
528 }
529 u, err := url.Parse(strings.TrimSpace(dest))
530 if err != nil {
531 return "", false
532 }
533 if requireExtension && !videoExtension(u.Path) {
534 return "", false
535 }
536 p.rawVideo = true
537 target, kind := p.target(dest)
538 if kind == dropped || strings.HasPrefix(target, "#") {
539 return "", false
540 }
541 if kind == external && !mediapolicy.Allows(u, p.videoHosts) {
542 return "", false
543 }
544 return target, true
545}
546
547func (p linkPolicy) standaloneVideo(n ast.Node, dest, label string) bool {
548 parent := n.Parent()
549 if parent == nil || parent.Kind() != ast.KindParagraph || parent.ChildCount() != 1 {
550 return false
551 }
552 u, err := url.Parse(dest)
553 if err != nil || !mediapolicy.Allows(u, p.videoHosts) {
554 return false
555 }
556 if !videoExtension(u.Path) && !githubAttachment(u) {
557 return false
558 }
559 src, ok := p.videoTarget(dest, false)
560 if !ok {
561 return false
562 }
563 parent.ReplaceChild(parent, n, &videoNode{src: src, alt: label})
564 return true
565}
566
567var kindVideo = ast.NewNodeKind("Video")
568
569type videoNode struct {
570 ast.BaseInline
571 src, alt string
572}
573
574func (n *videoNode) Kind() ast.NodeKind { return kindVideo }
575func (n *videoNode) Dump(source []byte, level int) { ast.DumpHelper(n, source, level, nil, nil) }
576
577func videoExtension(name string) bool {
578 switch strings.ToLower(path.Ext(name)) {
579 case ".mp4", ".webm", ".mov", ".m4v", ".ogv":
580 return true
581 default:
582 return false
583 }
584}
585
586// githubAttachment recognises only GitHub's extensionless attachment route,
587// not arbitrary extensionless pages on a host that also serves video files.
588func githubAttachment(u *url.URL) bool {
589 if !strings.EqualFold(u.Hostname(), "github.com") {
590 return false
591 }
592 id, ok := strings.CutPrefix(u.Path, "/user-attachments/assets/")
593 return ok && id != "" && id != "." && id != ".." && !strings.Contains(id, "/") && path.Ext(id) == ""
594}