1// SPDX-License-Identifier: MIT
2
3package template
4
5import (
6 "bytes"
7 "errors"
8 "fmt"
9 "io"
10 "math"
11 "regexp"
12 "strconv"
13 "strings"
14
15 "go.yaml.in/yaml/v3"
16)
17
18// MaxFileBytes is the largest document [ReadYAML] accepts: 1 MiB. The check
19// comes before any parsing, so that a hostile file costs nothing more than
20// its length; a file this size can still expand through aliases (see
21// [MaxNodes]).
22//
23// TODO(verify with recording): GitHub's own limit and message.
24const MaxFileBytes = 1 << 20
25
26// MaxNodes is the number of YAML nodes a document may expand to, counting
27// every use of an alias. The reference expands aliases without a limit, so a
28// small document of nested aliases can allocate gigabytes.
29const MaxNodes = 200_000
30
31type rkind int
32
33const (
34 rLiteral rkind = iota
35 rSeq
36 rMap
37)
38
39// rnode is a YAML node after alias resolution: the tree the template reader
40// walks.
41type rnode struct {
42 kind rkind
43 pos Pos
44 lit Token // rLiteral: a *String, *Number, *Boolean or *Null
45 items []*rnode
46 keys []*rnode // rMap: the keys, rLiteral nodes with string tokens
47 vals []*rnode // rMap: the values, parallel to keys
48 // quoted and block describe how a string scalar was written; they only
49 // matter to the position of expressions inside it.
50 quoted bool
51 block bool
52 // badKey is set on a key that is a collection.
53 badKey bool
54 // fault is set in runner mode (Context.RunnerYAML) on a node the runner's
55 // YAML reader refuses. The template reader stops with this message when it
56 // reaches the node, so the errors found before it in the document are kept.
57 fault string
58}
59
60// yamlResult is the outcome of reading a YAML document.
61type yamlResult struct {
62 root *rnode // nil for an empty document
63 errs []*Error
64 // after holds errors to record once the document is read (runner mode:
65 // the runner notices a second document only after reading the first).
66 after []*Error
67}
68
69type yamlBuilder struct {
70 file string
71 nodes int
72 errs []*Error
73 stack map[*yaml.Node]bool
74 over bool
75 // runnerYAML selects the runner's restrictions, see Context.RunnerYAML.
76 runnerYAML bool
77}
78
79// readYAMLDocument parses content with yaml.v3 and builds the tree. Syntax
80// errors are returned in errs, with root nil.
81func readYAMLDocument(file string, content []byte, runnerYAML bool) (res yamlResult) {
82 defer func() {
83 if r := recover(); r != nil {
84 res = yamlResult{errs: []*Error{{File: file, Raw: fmt.Sprintf("%v", r)}}}
85 }
86 }()
87 dec := yaml.NewDecoder(bytes.NewReader(content))
88 var doc yaml.Node
89 err := dec.Decode(&doc)
90 if errors.Is(err, io.EOF) {
91 return yamlResult{}
92 }
93 if err != nil {
94 return yamlResult{errs: []*Error{yamlError(file, err)}}
95 }
96 var after []*Error
97 var second yaml.Node
98 if err := dec.Decode(&second); err == nil {
99 if runnerYAML {
100 // The runner reads the first document, then fails on the event
101 // that is not the end of the stream.
102 after = []*Error{{File: file, fileOnly: true, Raw: "Expected stream end parse event"}}
103 } else {
104 e := &Error{File: file, Raw: "Source contains multiple documents; please use YAML.parseAllDocuments()"}
105 e.Pos = Pos{second.Line, second.Column}
106 e.fileOnly = true
107 e.Raw += fmt.Sprintf(" at line %d, column %d", second.Line, second.Column)
108 return yamlResult{errs: []*Error{e}}
109 }
110 } else if !errors.Is(err, io.EOF) {
111 return yamlResult{errs: []*Error{yamlError(file, err)}}
112 }
113 if len(doc.Content) == 0 {
114 return yamlResult{}
115 }
116 b := &yamlBuilder{file: file, stack: map[*yaml.Node]bool{}, runnerYAML: runnerYAML}
117 root := b.build(doc.Content[0], false)
118 if b.over {
119 return yamlResult{errs: b.errs}
120 }
121 return yamlResult{root: root, errs: b.errs, after: after}
122}
123
124var yamlLineRE = regexp.MustCompile(`^(?:yaml: )?(?:unmarshal errors:\n\s*)?(?:line (\d+): )?`)
125
126// yamlError converts an error of yaml.v3 ("yaml: line 3: did not find ...")
127// to an Error. The wording is yaml.v3's, not the JavaScript library's, and
128// only the line is known.
129func yamlError(file string, err error) *Error {
130 msg := err.Error()
131 e := &Error{File: file, fileOnly: true}
132 if m := yamlLineRE.FindStringSubmatch(msg); m != nil {
133 if m[1] != "" {
134 e.Pos.Line, _ = strconv.Atoi(m[1])
135 }
136 msg = msg[len(m[0]):]
137 }
138 e.Raw = msg
139 if e.Pos.Line > 0 {
140 e.Raw += fmt.Sprintf(" at line %d", e.Pos.Line)
141 }
142 return e
143}
144
145func (b *yamlBuilder) count() bool {
146 b.nodes++
147 if b.nodes > MaxNodes {
148 if !b.over {
149 b.over = true
150 b.errs = append(b.errs, &Error{File: b.file, Raw: fmt.Sprintf("The document is too large: it has more than %d nodes once aliases are expanded", MaxNodes)})
151 }
152 return false
153 }
154 return true
155}
156
157func (b *yamlBuilder) build(n *yaml.Node, asKey bool) *rnode {
158 r := b.build1(n, asKey)
159 if b.runnerYAML && r.fault == "" && n.Kind != yaml.AliasNode {
160 r.fault = runnerYAMLError(n)
161 }
162 return r
163}
164
165func (b *yamlBuilder) build1(n *yaml.Node, asKey bool) *rnode {
166 if b.over || !b.count() {
167 return &rnode{kind: rLiteral, lit: NewNull(Origin{})}
168 }
169 pos := Pos{n.Line, n.Column}
170 o := Origin{File: b.file, Pos: pos}
171 switch n.Kind {
172 case yaml.AliasNode:
173 target := n.Alias
174 if target == nil || b.stack[target] {
175 b.errs = append(b.errs, &Error{File: b.file, Pos: pos, Raw: fmt.Sprintf("Circular reference through the YAML alias '*%s'", n.Value)})
176 return &rnode{kind: rLiteral, pos: pos, lit: NewNull(o)}
177 }
178 b.stack[target] = true
179 r := b.build(target, asKey)
180 delete(b.stack, target)
181 return r
182 case yaml.SequenceNode:
183 if asKey {
184 return &rnode{kind: rLiteral, pos: pos, lit: NewString(o, ""), badKey: true}
185 }
186 r := &rnode{kind: rSeq, pos: pos}
187 b.stack[n] = true
188 for _, c := range n.Content {
189 r.items = append(r.items, b.build(c, false))
190 }
191 delete(b.stack, n)
192 return r
193 case yaml.MappingNode:
194 if asKey {
195 return &rnode{kind: rLiteral, pos: pos, lit: NewString(o, ""), badKey: true}
196 }
197 r := &rnode{kind: rMap, pos: pos}
198 b.stack[n] = true
199 for i := 0; i+1 < len(n.Content); i += 2 {
200 r.keys = append(r.keys, b.build(n.Content[i], true))
201 r.vals = append(r.vals, b.build(n.Content[i+1], false))
202 }
203 delete(b.stack, n)
204 return r
205 case yaml.ScalarNode:
206 r := &rnode{kind: rLiteral, pos: pos}
207 if asKey && !b.runnerYAML {
208 // Keys are always strings, whatever they look like.
209 r.lit = NewString(o, n.Value)
210 } else {
211 // The runner's YAML reader resolves a key like a value: "~" is
212 // null, "0x10" is 16, "TRUE" is true.
213 r.lit = resolveScalar(o, n, b.runnerYAML)
214 }
215 r.quoted = n.Style&(yaml.DoubleQuotedStyle|yaml.SingleQuotedStyle) != 0
216 r.block = n.Style&(yaml.LiteralStyle|yaml.FoldedStyle) != 0
217 return r
218 }
219 return &rnode{kind: rLiteral, pos: pos, lit: NewNull(o)}
220}
221
222// runnerYAMLError returns the message of the runner's YAML reader
223// (Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs) for a node
224// it refuses, or "". It checks anchors, and the tag and style of scalars.
225func runnerYAMLError(n *yaml.Node) string {
226 if n.Anchor != "" {
227 return fmt.Sprintf("Anchors are not currently supported. Remove the anchor '%s'", n.Anchor)
228 }
229 if n.Kind == yaml.ScalarNode && n.Style&(yaml.TaggedStyle|yaml.DoubleQuotedStyle|yaml.SingleQuotedStyle|yaml.LiteralStyle|yaml.FoldedStyle) == 0 {
230 if !fitsInt32(n.Value) {
231 return fmt.Sprintf("The value '%s' on line %d and column %d is invalid for the type ''", n.Value, n.Line, n.Column)
232 }
233 }
234 if n.Kind != yaml.ScalarNode || n.Style&yaml.TaggedStyle == 0 {
235 return ""
236 }
237 tag := n.LongTag()
238 const yamlTag = "tag:yaml.org,2002:"
239 // The runner's string tag is "tag:yaml.org,2002:string", which no YAML
240 // document spells (!!str is "tag:yaml.org,2002:str").
241 if tag == yamlTag+"string" {
242 return ""
243 }
244 plain := n.Style&(yaml.DoubleQuotedStyle|yaml.SingleQuotedStyle|yaml.LiteralStyle|yaml.FoldedStyle) == 0
245 if !plain {
246 style := "SingleQuoted"
247 switch {
248 case n.Style&yaml.DoubleQuotedStyle != 0:
249 style = "DoubleQuoted"
250 case n.Style&yaml.LiteralStyle != 0:
251 style = "Literal"
252 case n.Style&yaml.FoldedStyle != 0:
253 style = "Folded"
254 }
255 return fmt.Sprintf("The scalar style '%s' on line %d and column %d is not valid with the tag '%s'", style, n.Line, n.Column, tag)
256 }
257 switch tag {
258 case yamlTag + "bool", yamlTag + "float", yamlTag + "int", yamlTag + "null":
259 default:
260 return fmt.Sprintf("Unexpected tag '%s'", tag)
261 }
262 t := resolvePlain(Origin{}, n.Value, true)
263 ok := false
264 switch tag {
265 case yamlTag + "bool":
266 ok = t.Kind() == KindBoolean
267 case yamlTag + "null":
268 ok = t.Kind() == KindNull
269 case yamlTag + "float":
270 // MatchFloat knows no 0x or 0o form.
271 ok = t.Kind() == KindNumber && !reHex.MatchString(n.Value) && !reOct.MatchString(n.Value)
272 case yamlTag + "int":
273 ok = t.Kind() == KindNumber && (reInt.MatchString(n.Value) || reOct.MatchString(n.Value) || reHex.MatchString(n.Value)) && fitsInt32(n.Value)
274 }
275 if !ok {
276 return fmt.Sprintf("The value '%s' on line %d and column %d is invalid for the type '%s'", n.Value, n.Line, n.Column, tag)
277 }
278 return ""
279}
280
281// fitsInt32 reports whether a plain 0x or 0o integer is accepted by the
282// runner's reader: Int32.TryParse with AllowHexSpecifier (up to FFFFFFFF, read
283// as a negative number above 7FFFFFFF) and Convert.ToInt32(s, 8) (up to
284// 37777777777). Other values always fit.
285func fitsInt32(v string) bool {
286 var base uint64
287 switch {
288 case reHex.MatchString(v):
289 base = 16
290 case reOct.MatchString(v):
291 base = 8
292 default:
293 return true
294 }
295 _, ok := parseUint32(v[2:], base)
296 return ok
297}
298
299// parseUint32 reads digits in base 8 or 16, reporting false past 32 bits.
300func parseUint32(digits string, base uint64) (uint32, bool) {
301 var acc uint64
302 for _, c := range strings.ToLower(digits) {
303 d := uint64(c - '0')
304 if c >= 'a' {
305 d = uint64(c-'a') + 10
306 }
307 acc = acc*base + d
308 if acc > 1<<32-1 {
309 return 0, false
310 }
311 }
312 return uint32(acc), true
313}
314
315// resolveScalar gives a scalar its type the way the YAML 1.2 core schema of
316// the JavaScript yaml library does: only plain scalars are resolved, and an
317// explicit !!tag overrides the guess.
318func resolveScalar(o Origin, n *yaml.Node, runner bool) Token {
319 if n.Style&yaml.TaggedStyle != 0 {
320 switch n.ShortTag() {
321 case "!!str":
322 return NewString(o, n.Value)
323 case "!!null", "!!bool", "!!int", "!!float":
324 if t := resolvePlain(o, n.Value, runner); t.Kind() != KindString {
325 return t
326 }
327 if n.ShortTag() == "!!null" && n.Value == "" {
328 return NewNull(o)
329 }
330 }
331 return NewString(o, n.Value)
332 }
333 if n.Style&(yaml.DoubleQuotedStyle|yaml.SingleQuotedStyle|yaml.LiteralStyle|yaml.FoldedStyle) != 0 {
334 return NewString(o, n.Value)
335 }
336 return resolvePlain(o, n.Value, runner)
337}
338
339var (
340 reInt = regexp.MustCompile(`^[-+]?[0-9]+$`)
341 reOct = regexp.MustCompile(`^0o[0-7]+$`)
342 reHex = regexp.MustCompile(`^0x[0-9a-fA-F]+$`)
343 reFloat = regexp.MustCompile(`^[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?$`)
344 reInf = regexp.MustCompile(`^[-+]?\.(?:inf|Inf|INF)$`)
345 reNaN = regexp.MustCompile(`^\.(?:nan|NaN|NAN)$`)
346)
347
348// resolvePlain gives a plain scalar its type. In runner mode a 0x or 0o
349// integer is the 32-bit two's complement value the runner reads: 0xFFFFFFFF
350// is -1 and 0x80000000 is -2147483648.
351func resolvePlain(o Origin, s string, runner bool) Token {
352 switch s {
353 case "", "~", "null", "Null", "NULL":
354 return NewNull(o)
355 case "true", "True", "TRUE":
356 return NewBoolean(o, true)
357 case "false", "False", "FALSE":
358 return NewBoolean(o, false)
359 }
360 switch {
361 case reInt.MatchString(s):
362 f, _ := strconv.ParseFloat(s, 64)
363 return NewNumber(o, f)
364 case reOct.MatchString(s):
365 if u, ok := parseUint32(s[2:], 8); runner && ok {
366 return NewNumber(o, float64(int32(u)))
367 }
368 return NewNumber(o, parseBase(s[2:], 8))
369 case reHex.MatchString(s):
370 if u, ok := parseUint32(s[2:], 16); runner && ok {
371 return NewNumber(o, float64(int32(u)))
372 }
373 return NewNumber(o, parseBase(s[2:], 16))
374 case reFloat.MatchString(s):
375 f, _ := strconv.ParseFloat(s, 64)
376 return NewNumber(o, f)
377 case reInf.MatchString(s):
378 if s[0] == '-' {
379 return NewNumber(o, math.Inf(-1))
380 }
381 return NewNumber(o, math.Inf(1))
382 case reNaN.MatchString(s):
383 return NewNumber(o, math.NaN())
384 }
385 return NewString(o, s)
386}
387
388// parseBase converts digits in base 8 or 16 to a float64 without overflowing,
389// like JavaScript's parseInt.
390func parseBase(digits string, base int) float64 {
391 var f float64
392 for _, c := range strings.ToLower(digits) {
393 d := int(c - '0')
394 if c >= 'a' {
395 d = int(c-'a') + 10
396 }
397 f = f*float64(base) + float64(d)
398 }
399 return f
400}