detached 1facd52

feat(expr): add the lexer

commit
1facd5247420c7ccb5320fc9e9c35ecfe65eccd6
author
brice · · 2026-10-12 14:46 UTC
parent
5904904
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
2 files · +466 -0
Unified
UnifiedSplit
A expr/errors.go +107 -0
@@ -0,0 +1,107 @@
1+// SPDX-License-Identifier: MIT
2+
3+package expr
4+
5+import (
6+ "fmt"
7+)
8+
9+// Limits of the reference implementation.
10+const (
11+ // MaxParserDepth is the deepest nesting of groupings, function arguments,
12+ // index accesses, negations and logical operators the parser accepts.
13+ MaxParserDepth = 50
14+ // MaxExpressionLength is the longest expression accepted, counted in
15+ // UTF-16 code units like the JavaScript string length it mirrors.
16+ MaxExpressionLength = 21000
17+)
18+
19+// ErrorType classifies an [Error].
20+type ErrorType int
21+
22+// The error types of the reference implementation, in its order.
23+const (
24+ ErrorUnexpectedSymbol ErrorType = iota
25+ // ErrorUnrecognizedNamedValue is declared by the reference but never
26+ // produced; ErrorUnrecognizedContext carries the same text.
27+ ErrorUnrecognizedNamedValue
28+ ErrorUnexpectedEndOfExpression
29+ ErrorExceededMaxDepth
30+ ErrorExceededMaxLength
31+ ErrorTooFewParameters
32+ ErrorTooManyParameters
33+ ErrorEvenParameters
34+ ErrorUnrecognizedContext
35+ ErrorUnrecognizedFunction
36+)
37+
38+func (t ErrorType) description() string {
39+ switch t {
40+ case ErrorUnexpectedEndOfExpression:
41+ return "Unexpected end of expression"
42+ case ErrorUnexpectedSymbol:
43+ return "Unexpected symbol"
44+ case ErrorUnrecognizedNamedValue:
45+ return "Unrecognized named-value"
46+ case ErrorExceededMaxDepth:
47+ return fmt.Sprintf("Exceeded max expression depth %d", MaxParserDepth)
48+ case ErrorExceededMaxLength:
49+ return fmt.Sprintf("Exceeded max expression length %d", MaxExpressionLength)
50+ case ErrorTooFewParameters:
51+ return "Too few parameters supplied"
52+ case ErrorTooManyParameters:
53+ return "Too many parameters supplied"
54+ case ErrorEvenParameters:
55+ return "Even number of parameters supplied, requires an odd number of parameters"
56+ case ErrorUnrecognizedContext:
57+ return "Unrecognized named-value"
58+ case ErrorUnrecognizedFunction:
59+ return "Unrecognized function"
60+ }
61+ return "Unknown error"
62+}
63+
64+// Position is a zero-based line and column in an expression. Columns count
65+// UTF-16 code units, as in the reference implementation.
66+type Position struct {
67+ Line int
68+ Column int
69+}
70+
71+// Error is a lexing or parsing error. Its message is exactly the one the
72+// reference implementation produces.
73+type Error struct {
74+ Type ErrorType
75+ // Message is the full text, e.g. "Unexpected symbol: ')'".
76+ Message string
77+ // Pos is where the offending token starts.
78+ Pos Position
79+ // Token is the text of the offending token.
80+ Token string
81+}
82+
83+func (e *Error) Error() string { return e.Message }
84+
85+func newTokenError(t ErrorType, tok Token) *Error {
86+ return &Error{
87+ Type: t,
88+ Message: t.description() + ": '" + tokenString(tok) + "'",
89+ Pos: tok.Range.Start,
90+ Token: tokenString(tok),
91+ }
92+}
93+
94+// EvalError is an error raised while evaluating an expression: a failing
95+// function, such as fromJSON on invalid input, or an invalid format string.
96+type EvalError struct {
97+ Message string
98+ // Err is the underlying error, if any.
99+ Err error
100+}
101+
102+func (e *EvalError) Error() string { return e.Message }
103+func (e *EvalError) Unwrap() error { return e.Err }
104+
105+func evalErrorf(format string, a ...any) *EvalError {
106+ return &EvalError{Message: fmt.Sprintf(format, a...)}
107+}
@@ -0,0 +1,107 @@
1+// SPDX-License-Identifier: MIT
2+
3+package expr
4+
5+import (
6+ "fmt"
7+)
8+
9+// Limits of the reference implementation.
10+const (
11+ // MaxParserDepth is the deepest nesting of groupings, function arguments,
12+ // index accesses, negations and logical operators the parser accepts.
13+ MaxParserDepth = 50
14+ // MaxExpressionLength is the longest expression accepted, counted in
15+ // UTF-16 code units like the JavaScript string length it mirrors.
16+ MaxExpressionLength = 21000
17+)
18+
19+// ErrorType classifies an [Error].
20+type ErrorType int
21+
22+// The error types of the reference implementation, in its order.
23+const (
24+ ErrorUnexpectedSymbol ErrorType = iota
25+ // ErrorUnrecognizedNamedValue is declared by the reference but never
26+ // produced; ErrorUnrecognizedContext carries the same text.
27+ ErrorUnrecognizedNamedValue
28+ ErrorUnexpectedEndOfExpression
29+ ErrorExceededMaxDepth
30+ ErrorExceededMaxLength
31+ ErrorTooFewParameters
32+ ErrorTooManyParameters
33+ ErrorEvenParameters
34+ ErrorUnrecognizedContext
35+ ErrorUnrecognizedFunction
36+)
37+
38+func (t ErrorType) description() string {
39+ switch t {
40+ case ErrorUnexpectedEndOfExpression:
41+ return "Unexpected end of expression"
42+ case ErrorUnexpectedSymbol:
43+ return "Unexpected symbol"
44+ case ErrorUnrecognizedNamedValue:
45+ return "Unrecognized named-value"
46+ case ErrorExceededMaxDepth:
47+ return fmt.Sprintf("Exceeded max expression depth %d", MaxParserDepth)
48+ case ErrorExceededMaxLength:
49+ return fmt.Sprintf("Exceeded max expression length %d", MaxExpressionLength)
50+ case ErrorTooFewParameters:
51+ return "Too few parameters supplied"
52+ case ErrorTooManyParameters:
53+ return "Too many parameters supplied"
54+ case ErrorEvenParameters:
55+ return "Even number of parameters supplied, requires an odd number of parameters"
56+ case ErrorUnrecognizedContext:
57+ return "Unrecognized named-value"
58+ case ErrorUnrecognizedFunction:
59+ return "Unrecognized function"
60+ }
61+ return "Unknown error"
62+}
63+
64+// Position is a zero-based line and column in an expression. Columns count
65+// UTF-16 code units, as in the reference implementation.
66+type Position struct {
67+ Line int
68+ Column int
69+}
70+
71+// Error is a lexing or parsing error. Its message is exactly the one the
72+// reference implementation produces.
73+type Error struct {
74+ Type ErrorType
75+ // Message is the full text, e.g. "Unexpected symbol: ')'".
76+ Message string
77+ // Pos is where the offending token starts.
78+ Pos Position
79+ // Token is the text of the offending token.
80+ Token string
81+}
82+
83+func (e *Error) Error() string { return e.Message }
84+
85+func newTokenError(t ErrorType, tok Token) *Error {
86+ return &Error{
87+ Type: t,
88+ Message: t.description() + ": '" + tokenString(tok) + "'",
89+ Pos: tok.Range.Start,
90+ Token: tokenString(tok),
91+ }
92+}
93+
94+// EvalError is an error raised while evaluating an expression: a failing
95+// function, such as fromJSON on invalid input, or an invalid format string.
96+type EvalError struct {
97+ Message string
98+ // Err is the underlying error, if any.
99+ Err error
100+}
101+
102+func (e *EvalError) Error() string { return e.Message }
103+func (e *EvalError) Unwrap() error { return e.Err }
104+
105+func evalErrorf(format string, a ...any) *EvalError {
106+ return &EvalError{Message: fmt.Sprintf(format, a...)}
107+}
A expr/lexer.go +359 -0
@@ -0,0 +1,359 @@
1+// SPDX-License-Identifier: MIT
2+
3+package expr
4+
5+import (
6+ "fmt"
7+ "math"
8+ "strings"
9+ "unicode/utf16"
10+)
11+
12+// TokenType is the kind of a lexical token.
13+type TokenType int
14+
15+// The token types, in the order of the reference implementation.
16+const (
17+ TokenUnknown TokenType = iota
18+ TokenLeftParen
19+ TokenRightParen
20+ TokenLeftBracket
21+ TokenRightBracket
22+ TokenComma
23+ TokenDot
24+ TokenBang
25+ TokenBangEqual
26+ TokenEqualEqual
27+ TokenGreater
28+ TokenGreaterEqual
29+ TokenLess
30+ TokenLessEqual
31+ TokenAnd
32+ TokenOr
33+ TokenStar
34+ TokenNumber
35+ TokenString
36+ TokenIdentifier
37+ TokenTrue
38+ TokenFalse
39+ TokenNull
40+ TokenEOF
41+)
42+
43+var tokenTypeNames = [...]string{
44+ "UNKNOWN", "LEFT_PAREN", "RIGHT_PAREN", "LEFT_BRACKET", "RIGHT_BRACKET",
45+ "COMMA", "DOT", "BANG", "BANG_EQUAL", "EQUAL_EQUAL", "GREATER",
46+ "GREATER_EQUAL", "LESS", "LESS_EQUAL", "AND", "OR", "STAR", "NUMBER",
47+ "STRING", "IDENTIFIER", "TRUE", "FALSE", "NULL", "EOF",
48+}
49+
50+func (t TokenType) String() string {
51+ if t >= 0 && int(t) < len(tokenTypeNames) {
52+ return tokenTypeNames[t]
53+ }
54+ return fmt.Sprintf("TokenType(%d)", int(t))
55+}
56+
57+// Range is the extent of a token. Both positions are on the line where the
58+// token ends, as in the reference implementation.
59+type Range struct {
60+ Start, End Position
61+}
62+
63+// Token is a lexical token.
64+type Token struct {
65+ Type TokenType
66+ Lexeme string
67+ Range Range
68+ // Value is the number of a TokenNumber, as a [Number], or the unescaped
69+ // text of a TokenString, as a [String]; nil for other tokens.
70+ Value Value
71+}
72+
73+// tokenString is the text an error message shows for a token.
74+func tokenString(t Token) string {
75+ switch t.Type {
76+ case TokenEOF:
77+ return "EOF"
78+ case TokenString:
79+ return string(t.Value.(String))
80+ }
81+ return t.Lexeme
82+}
83+
84+// Lex splits an expression into tokens. The result always ends with a
85+// TokenEOF token.
86+func Lex(input string) ([]Token, error) {
87+ units := utf16.Encode([]rune(input))
88+ if len(units) > MaxExpressionLength {
89+ // The reference throws a bare Error("ErrorExceededMaxLength") here,
90+ // not the descriptive ExpressionError the parser would build.
91+ return nil, &Error{Type: ErrorExceededMaxLength, Message: "ErrorExceededMaxLength"}
92+ }
93+ lx := &lexer{input: string(utf16.Decode(units)), units: units}
94+ if err := lx.lex(); err != nil {
95+ return nil, err
96+ }
97+ return lx.tokens, nil
98+}
99+
100+// lexer is a line-by-line port of Lexer in lexer.js. It works on UTF-16 code
101+// units so that positions match the reference.
102+type lexer struct {
103+ input string
104+ units []uint16
105+ start int
106+ offset int
107+ line int
108+ lastLineOffset int
109+ tokens []Token
110+}
111+
112+func (l *lexer) lex() error {
113+ for !l.atEnd() {
114+ l.start = l.offset
115+ c := l.next()
116+ var err error
117+ switch c {
118+ case '(':
119+ l.addToken(TokenLeftParen, nil)
120+ case ')':
121+ l.addToken(TokenRightParen, nil)
122+ case '[':
123+ l.addToken(TokenLeftBracket, nil)
124+ case ']':
125+ l.addToken(TokenRightBracket, nil)
126+ case ',':
127+ l.addToken(TokenComma, nil)
128+ case '.':
129+ p := l.previous()
130+ if p != TokenIdentifier && p != TokenRightBracket && p != TokenRightParen && p != TokenStar {
131+ err = l.consumeNumber()
132+ } else {
133+ l.addToken(TokenDot, nil)
134+ }
135+ case '-', '+':
136+ err = l.consumeNumber()
137+ case '!':
138+ l.addToken(l.pick('=', TokenBangEqual, TokenBang), nil)
139+ case '=':
140+ if !l.match('=') {
141+ // Illegal; continue reading until a boundary and report it.
142+ err = l.consumeIdentifier()
143+ break
144+ }
145+ l.addToken(TokenEqualEqual, nil)
146+ case '<':
147+ l.addToken(l.pick('=', TokenLessEqual, TokenLess), nil)
148+ case '>':
149+ l.addToken(l.pick('=', TokenGreaterEqual, TokenGreater), nil)
150+ case '&':
151+ if !l.match('&') {
152+ err = l.consumeIdentifier()
153+ break
154+ }
155+ l.addToken(TokenAnd, nil)
156+ case '|':
157+ if !l.match('|') {
158+ err = l.consumeIdentifier()
159+ break
160+ }
161+ l.addToken(TokenOr, nil)
162+ case '*':
163+ l.addToken(TokenStar, nil)
164+ case ' ', '\r', '\t':
165+ // Whitespace is ignored. Only these three characters and the
166+ // newline are: other Unicode spaces (U+00A0, \v, \f...) are not
167+ // skipped here and fall through to the identifier rule.
168+ case '\n':
169+ l.line++
170+ l.lastLineOffset = l.offset
171+ case '\'':
172+ err = l.consumeString()
173+ default:
174+ if c >= '0' && c <= '9' {
175+ err = l.consumeNumber()
176+ } else {
177+ err = l.consumeIdentifier()
178+ }
179+ }
180+ if err != nil {
181+ return err
182+ }
183+ }
184+ l.tokens = append(l.tokens, Token{Type: TokenEOF, Range: l.rng()})
185+ return nil
186+}
187+
188+func (l *lexer) pos() Position {
189+ return Position{Line: l.line, Column: l.start - l.lastLineOffset}
190+}
191+
192+func (l *lexer) rng() Range {
193+ return Range{
194+ Start: l.pos(),
195+ End: Position{Line: l.line, Column: l.offset - l.lastLineOffset},
196+ }
197+}
198+
199+func (l *lexer) atEnd() bool { return l.offset >= len(l.units) }
200+
201+// peek returns the next unit without consuming it, or 0 at the end.
202+func (l *lexer) peek() uint16 {
203+ if l.atEnd() {
204+ return 0
205+ }
206+ return l.units[l.offset]
207+}
208+
209+func (l *lexer) peekNext() uint16 {
210+ if l.offset+1 >= len(l.units) {
211+ return 0
212+ }
213+ return l.units[l.offset+1]
214+}
215+
216+func (l *lexer) previous() TokenType {
217+ if len(l.tokens) == 0 {
218+ return TokenEOF
219+ }
220+ return l.tokens[len(l.tokens)-1].Type
221+}
222+
223+func (l *lexer) next() uint16 {
224+ c := l.units[l.offset]
225+ l.offset++
226+ return c
227+}
228+
229+func (l *lexer) match(expected uint16) bool {
230+ if l.atEnd() || l.units[l.offset] != expected {
231+ return false
232+ }
233+ l.offset++
234+ return true
235+}
236+
237+// pick consumes expected if it is next and returns yes, otherwise no.
238+func (l *lexer) pick(expected uint16, yes, no TokenType) TokenType {
239+ if l.match(expected) {
240+ return yes
241+ }
242+ return no
243+}
244+
245+func (l *lexer) substring(from, to int) string {
246+ return string(utf16.Decode(l.units[from:to]))
247+}
248+
249+func (l *lexer) addToken(t TokenType, v Value) {
250+ l.tokens = append(l.tokens, Token{
251+ Type: t,
252+ Lexeme: l.substring(l.start, l.offset),
253+ Range: l.rng(),
254+ Value: v,
255+ })
256+}
257+
258+func (l *lexer) unexpected(lexeme string) *Error {
259+ return &Error{
260+ Type: ErrorUnexpectedSymbol,
261+ Message: fmt.Sprintf("Unexpected symbol: '%s'. Located at position %d within expression: %s", lexeme, l.start+1, l.input),
262+ Pos: l.pos(),
263+ Token: lexeme,
264+ }
265+}
266+
267+func (l *lexer) consumeNumber() error {
268+ for !l.atEnd() && (!isBoundary(l.peek()) || l.peek() == '.') {
269+ l.next()
270+ }
271+ lexeme := l.substring(l.start, l.offset)
272+ v := jsToNumber(lexeme)
273+ if math.IsNaN(v) {
274+ return l.unexpected(lexeme)
275+ }
276+ l.addToken(TokenNumber, Number(v))
277+ return nil
278+}
279+
280+func (l *lexer) consumeString() error {
281+ for (l.peek() != '\'' || l.peekNext() == '\'') && !l.atEnd() {
282+ if l.peek() == '\n' {
283+ l.line++
284+ }
285+ if l.peek() == '\'' && l.peekNext() == '\'' {
286+ // Escaped quote: consume the first of the pair here.
287+ l.next()
288+ }
289+ l.next()
290+ }
291+ if l.atEnd() {
292+ return l.unexpected(l.substring(l.start, len(l.units)))
293+ }
294+ l.next() // closing quote
295+ value := l.substring(l.start+1, l.offset-1)
296+ // The reference calls value.replace("''", "'") with a string pattern,
297+ // which replaces only the FIRST doubled quote: 'a''b''c' is a'b''c.
298+ // GitHub's C# lexer unescapes every pair. Followed as written; see
299+ // PORTING.md.
300+ value = strings.Replace(value, "''", "'", 1)
301+ l.addToken(TokenString, String(value))
302+ return nil
303+}
304+
305+func (l *lexer) consumeIdentifier() error {
306+ for !l.atEnd() && !isBoundary(l.peek()) {
307+ l.next()
308+ }
309+ tokenType := TokenIdentifier
310+ var tokenValue Value
311+ lexeme := l.substring(l.start, l.offset)
312+ if l.previous() != TokenDot {
313+ switch lexeme {
314+ case "true":
315+ tokenType = TokenTrue
316+ case "false":
317+ tokenType = TokenFalse
318+ case "null":
319+ tokenType = TokenNull
320+ case "NaN":
321+ tokenType = TokenNumber
322+ tokenValue = Number(nan)
323+ case "Infinity":
324+ tokenType = TokenNumber
325+ tokenValue = Number(posInf)
326+ }
327+ }
328+ if !isLegalIdentifier(lexeme) {
329+ return l.unexpected(lexeme)
330+ }
331+ l.addToken(tokenType, tokenValue)
332+ return nil
333+}
334+
335+// isBoundary reports whether c ends a number or identifier.
336+func isBoundary(c uint16) bool {
337+ switch c {
338+ case '(', '[', ')', ']', ',', '.', '!', '>', '<', '=', '&', '|':
339+ return true
340+ }
341+ return isJSSpace(rune(c))
342+}
343+
344+// isLegalIdentifier reports whether s matches [A-Za-z_][A-Za-z0-9_-]*.
345+func isLegalIdentifier(s string) bool {
346+ if s == "" {
347+ return false
348+ }
349+ for i := 0; i < len(s); i++ {
350+ c := s[i]
351+ switch {
352+ case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c == '_':
353+ case i > 0 && (c >= '0' && c <= '9' || c == '-'):
354+ default:
355+ return false
356+ }
357+ }
358+ return true
359+}
@@ -0,0 +1,359 @@
1+// SPDX-License-Identifier: MIT
2+
3+package expr
4+
5+import (
6+ "fmt"
7+ "math"
8+ "strings"
9+ "unicode/utf16"
10+)
11+
12+// TokenType is the kind of a lexical token.
13+type TokenType int
14+
15+// The token types, in the order of the reference implementation.
16+const (
17+ TokenUnknown TokenType = iota
18+ TokenLeftParen
19+ TokenRightParen
20+ TokenLeftBracket
21+ TokenRightBracket
22+ TokenComma
23+ TokenDot
24+ TokenBang
25+ TokenBangEqual
26+ TokenEqualEqual
27+ TokenGreater
28+ TokenGreaterEqual
29+ TokenLess
30+ TokenLessEqual
31+ TokenAnd
32+ TokenOr
33+ TokenStar
34+ TokenNumber
35+ TokenString
36+ TokenIdentifier
37+ TokenTrue
38+ TokenFalse
39+ TokenNull
40+ TokenEOF
41+)
42+
43+var tokenTypeNames = [...]string{
44+ "UNKNOWN", "LEFT_PAREN", "RIGHT_PAREN", "LEFT_BRACKET", "RIGHT_BRACKET",
45+ "COMMA", "DOT", "BANG", "BANG_EQUAL", "EQUAL_EQUAL", "GREATER",
46+ "GREATER_EQUAL", "LESS", "LESS_EQUAL", "AND", "OR", "STAR", "NUMBER",
47+ "STRING", "IDENTIFIER", "TRUE", "FALSE", "NULL", "EOF",
48+}
49+
50+func (t TokenType) String() string {
51+ if t >= 0 && int(t) < len(tokenTypeNames) {
52+ return tokenTypeNames[t]
53+ }
54+ return fmt.Sprintf("TokenType(%d)", int(t))
55+}
56+
57+// Range is the extent of a token. Both positions are on the line where the
58+// token ends, as in the reference implementation.
59+type Range struct {
60+ Start, End Position
61+}
62+
63+// Token is a lexical token.
64+type Token struct {
65+ Type TokenType
66+ Lexeme string
67+ Range Range
68+ // Value is the number of a TokenNumber, as a [Number], or the unescaped
69+ // text of a TokenString, as a [String]; nil for other tokens.
70+ Value Value
71+}
72+
73+// tokenString is the text an error message shows for a token.
74+func tokenString(t Token) string {
75+ switch t.Type {
76+ case TokenEOF:
77+ return "EOF"
78+ case TokenString:
79+ return string(t.Value.(String))
80+ }
81+ return t.Lexeme
82+}
83+
84+// Lex splits an expression into tokens. The result always ends with a
85+// TokenEOF token.
86+func Lex(input string) ([]Token, error) {
87+ units := utf16.Encode([]rune(input))
88+ if len(units) > MaxExpressionLength {
89+ // The reference throws a bare Error("ErrorExceededMaxLength") here,
90+ // not the descriptive ExpressionError the parser would build.
91+ return nil, &Error{Type: ErrorExceededMaxLength, Message: "ErrorExceededMaxLength"}
92+ }
93+ lx := &lexer{input: string(utf16.Decode(units)), units: units}
94+ if err := lx.lex(); err != nil {
95+ return nil, err
96+ }
97+ return lx.tokens, nil
98+}
99+
100+// lexer is a line-by-line port of Lexer in lexer.js. It works on UTF-16 code
101+// units so that positions match the reference.
102+type lexer struct {
103+ input string
104+ units []uint16
105+ start int
106+ offset int
107+ line int
108+ lastLineOffset int
109+ tokens []Token
110+}
111+
112+func (l *lexer) lex() error {
113+ for !l.atEnd() {
114+ l.start = l.offset
115+ c := l.next()
116+ var err error
117+ switch c {
118+ case '(':
119+ l.addToken(TokenLeftParen, nil)
120+ case ')':
121+ l.addToken(TokenRightParen, nil)
122+ case '[':
123+ l.addToken(TokenLeftBracket, nil)
124+ case ']':
125+ l.addToken(TokenRightBracket, nil)
126+ case ',':
127+ l.addToken(TokenComma, nil)
128+ case '.':
129+ p := l.previous()
130+ if p != TokenIdentifier && p != TokenRightBracket && p != TokenRightParen && p != TokenStar {
131+ err = l.consumeNumber()
132+ } else {
133+ l.addToken(TokenDot, nil)
134+ }
135+ case '-', '+':
136+ err = l.consumeNumber()
137+ case '!':
138+ l.addToken(l.pick('=', TokenBangEqual, TokenBang), nil)
139+ case '=':
140+ if !l.match('=') {
141+ // Illegal; continue reading until a boundary and report it.
142+ err = l.consumeIdentifier()
143+ break
144+ }
145+ l.addToken(TokenEqualEqual, nil)
146+ case '<':
147+ l.addToken(l.pick('=', TokenLessEqual, TokenLess), nil)
148+ case '>':
149+ l.addToken(l.pick('=', TokenGreaterEqual, TokenGreater), nil)
150+ case '&':
151+ if !l.match('&') {
152+ err = l.consumeIdentifier()
153+ break
154+ }
155+ l.addToken(TokenAnd, nil)
156+ case '|':
157+ if !l.match('|') {
158+ err = l.consumeIdentifier()
159+ break
160+ }
161+ l.addToken(TokenOr, nil)
162+ case '*':
163+ l.addToken(TokenStar, nil)
164+ case ' ', '\r', '\t':
165+ // Whitespace is ignored. Only these three characters and the
166+ // newline are: other Unicode spaces (U+00A0, \v, \f...) are not
167+ // skipped here and fall through to the identifier rule.
168+ case '\n':
169+ l.line++
170+ l.lastLineOffset = l.offset
171+ case '\'':
172+ err = l.consumeString()
173+ default:
174+ if c >= '0' && c <= '9' {
175+ err = l.consumeNumber()
176+ } else {
177+ err = l.consumeIdentifier()
178+ }
179+ }
180+ if err != nil {
181+ return err
182+ }
183+ }
184+ l.tokens = append(l.tokens, Token{Type: TokenEOF, Range: l.rng()})
185+ return nil
186+}
187+
188+func (l *lexer) pos() Position {
189+ return Position{Line: l.line, Column: l.start - l.lastLineOffset}
190+}
191+
192+func (l *lexer) rng() Range {
193+ return Range{
194+ Start: l.pos(),
195+ End: Position{Line: l.line, Column: l.offset - l.lastLineOffset},
196+ }
197+}
198+
199+func (l *lexer) atEnd() bool { return l.offset >= len(l.units) }
200+
201+// peek returns the next unit without consuming it, or 0 at the end.
202+func (l *lexer) peek() uint16 {
203+ if l.atEnd() {
204+ return 0
205+ }
206+ return l.units[l.offset]
207+}
208+
209+func (l *lexer) peekNext() uint16 {
210+ if l.offset+1 >= len(l.units) {
211+ return 0
212+ }
213+ return l.units[l.offset+1]
214+}
215+
216+func (l *lexer) previous() TokenType {
217+ if len(l.tokens) == 0 {
218+ return TokenEOF
219+ }
220+ return l.tokens[len(l.tokens)-1].Type
221+}
222+
223+func (l *lexer) next() uint16 {
224+ c := l.units[l.offset]
225+ l.offset++
226+ return c
227+}
228+
229+func (l *lexer) match(expected uint16) bool {
230+ if l.atEnd() || l.units[l.offset] != expected {
231+ return false
232+ }
233+ l.offset++
234+ return true
235+}
236+
237+// pick consumes expected if it is next and returns yes, otherwise no.
238+func (l *lexer) pick(expected uint16, yes, no TokenType) TokenType {
239+ if l.match(expected) {
240+ return yes
241+ }
242+ return no
243+}
244+
245+func (l *lexer) substring(from, to int) string {
246+ return string(utf16.Decode(l.units[from:to]))
247+}
248+
249+func (l *lexer) addToken(t TokenType, v Value) {
250+ l.tokens = append(l.tokens, Token{
251+ Type: t,
252+ Lexeme: l.substring(l.start, l.offset),
253+ Range: l.rng(),
254+ Value: v,
255+ })
256+}
257+
258+func (l *lexer) unexpected(lexeme string) *Error {
259+ return &Error{
260+ Type: ErrorUnexpectedSymbol,
261+ Message: fmt.Sprintf("Unexpected symbol: '%s'. Located at position %d within expression: %s", lexeme, l.start+1, l.input),
262+ Pos: l.pos(),
263+ Token: lexeme,
264+ }
265+}
266+
267+func (l *lexer) consumeNumber() error {
268+ for !l.atEnd() && (!isBoundary(l.peek()) || l.peek() == '.') {
269+ l.next()
270+ }
271+ lexeme := l.substring(l.start, l.offset)
272+ v := jsToNumber(lexeme)
273+ if math.IsNaN(v) {
274+ return l.unexpected(lexeme)
275+ }
276+ l.addToken(TokenNumber, Number(v))
277+ return nil
278+}
279+
280+func (l *lexer) consumeString() error {
281+ for (l.peek() != '\'' || l.peekNext() == '\'') && !l.atEnd() {
282+ if l.peek() == '\n' {
283+ l.line++
284+ }
285+ if l.peek() == '\'' && l.peekNext() == '\'' {
286+ // Escaped quote: consume the first of the pair here.
287+ l.next()
288+ }
289+ l.next()
290+ }
291+ if l.atEnd() {
292+ return l.unexpected(l.substring(l.start, len(l.units)))
293+ }
294+ l.next() // closing quote
295+ value := l.substring(l.start+1, l.offset-1)
296+ // The reference calls value.replace("''", "'") with a string pattern,
297+ // which replaces only the FIRST doubled quote: 'a''b''c' is a'b''c.
298+ // GitHub's C# lexer unescapes every pair. Followed as written; see
299+ // PORTING.md.
300+ value = strings.Replace(value, "''", "'", 1)
301+ l.addToken(TokenString, String(value))
302+ return nil
303+}
304+
305+func (l *lexer) consumeIdentifier() error {
306+ for !l.atEnd() && !isBoundary(l.peek()) {
307+ l.next()
308+ }
309+ tokenType := TokenIdentifier
310+ var tokenValue Value
311+ lexeme := l.substring(l.start, l.offset)
312+ if l.previous() != TokenDot {
313+ switch lexeme {
314+ case "true":
315+ tokenType = TokenTrue
316+ case "false":
317+ tokenType = TokenFalse
318+ case "null":
319+ tokenType = TokenNull
320+ case "NaN":
321+ tokenType = TokenNumber
322+ tokenValue = Number(nan)
323+ case "Infinity":
324+ tokenType = TokenNumber
325+ tokenValue = Number(posInf)
326+ }
327+ }
328+ if !isLegalIdentifier(lexeme) {
329+ return l.unexpected(lexeme)
330+ }
331+ l.addToken(tokenType, tokenValue)
332+ return nil
333+}
334+
335+// isBoundary reports whether c ends a number or identifier.
336+func isBoundary(c uint16) bool {
337+ switch c {
338+ case '(', '[', ')', ']', ',', '.', '!', '>', '<', '=', '&', '|':
339+ return true
340+ }
341+ return isJSSpace(rune(c))
342+}
343+
344+// isLegalIdentifier reports whether s matches [A-Za-z_][A-Za-z0-9_-]*.
345+func isLegalIdentifier(s string) bool {
346+ if s == "" {
347+ return false
348+ }
349+ for i := 0; i < len(s); i++ {
350+ c := s[i]
351+ switch {
352+ case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c == '_':
353+ case i > 0 && (c >= '0' && c <= '9' || c == '-'):
354+ default:
355+ return false
356+ }
357+ }
358+ return true
359+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 1

Tags 3

↑↓ moveenter openesc close