1package css
2
3import (
4 "strings"
5 "unicode"
6 "unicode/utf8"
7)
8
9// Token is a CSS syntax token. Raw preserves the original spelling for value parsing.
10type Token struct {
11 Kind, Raw, Value string
12 Line, Col int
13}
14
15// Tokenize scans CSS without panicking on malformed UTF-8 or unterminated constructs.
16func Tokenize(src string) []Token {
17 var out []Token
18 line, col := 1, 1
19 take := func(n int) string {
20 s := src[:n]
21 for _, r := range s {
22 if r == '\n' {
23 line++
24 col = 1
25 } else {
26 col++
27 }
28 }
29 src = src[n:]
30 return s
31 }
32 emit := func(k, s, v string, l, c int) { out = append(out, Token{k, s, v, l, c}) }
33 for len(src) > 0 {
34 l, c := line, col
35 if strings.HasPrefix(src, "/*") {
36 i := strings.Index(src[2:], "*/")
37 if i < 0 {
38 take(len(src))
39 } else {
40 take(i + 4)
41 }
42 // Comments separate tokens; raw() must preserve that separation.
43 emit("space", " ", " ", l, c)
44 continue
45 }
46 r, n := utf8.DecodeRuneInString(src)
47 if unicode.IsSpace(r) {
48 i := n
49 for i < len(src) {
50 rr, nn := utf8.DecodeRuneInString(src[i:])
51 if !unicode.IsSpace(rr) {
52 break
53 }
54 i += nn
55 }
56 emit("space", take(i), " ", l, c)
57 continue
58 }
59 if r == '\'' || r == '"' {
60 quote := r
61 i := n
62 var val strings.Builder
63 bad := false
64 closed := false
65 for i < len(src) {
66 rr, nn := utf8.DecodeRuneInString(src[i:])
67 if rr == quote {
68 i += nn
69 closed = true
70 break
71 }
72 if rr == '\n' {
73 bad = true
74 break
75 }
76 if rr == '\\' {
77 i += nn
78 if i < len(src) {
79 er, en := escape(src[i:])
80 val.WriteRune(er)
81 i += en
82 }
83 continue
84 }
85 val.WriteRune(rr)
86 i += nn
87 }
88 if !closed {
89 bad = true
90 }
91 s := take(i)
92 k := "string"
93 if bad {
94 k = "bad-string"
95 }
96 emit(k, s, val.String(), l, c)
97 continue
98 }
99 if r == '#' && len(src) > n && identStart(src[n:]) {
100 i := n + identLen(src[n:])
101 s := take(i)
102 emit("hash", s, unescapeIdent(s[n:]), l, c)
103 continue
104 }
105 if numberStart(src) {
106 i := numberLen(src)
107 k := "number"
108 if i < len(src) && src[i] == '%' {
109 i++
110 k = "percentage"
111 } else if i < len(src) && identStart(src[i:]) {
112 i += identLen(src[i:])
113 k = "dimension"
114 }
115 s := take(i)
116 emit(k, s, s, l, c)
117 continue
118 }
119 if identStart(src) {
120 i := identLen(src)
121 k := "ident"
122 if i < len(src) && src[i] == '(' {
123 i++
124 k = "function"
125 }
126 s := take(i)
127 value := s
128 if k == "ident" || k == "function" {
129 value = unescapeIdent(strings.TrimSuffix(s, "("))
130 }
131 if k == "function" && strings.EqualFold(value, "url") {
132 // CSS Syntax: unquoted url() consumes to ')' or EOF, recovering bad-url.
133 rest := src
134 off := 0
135 for off < len(rest) && (rest[off] == ' ' || rest[off] == '\n' || rest[off] == '\t') {
136 off++
137 }
138 if off < len(rest) && rest[off] != '\'' && rest[off] != '"' {
139 start := off
140 bad := false
141 for off < len(rest) && rest[off] != ')' {
142 if rest[off] == '"' || rest[off] == '\'' || rest[off] == '(' || rest[off] == '\n' {
143 bad = true
144 }
145 if rest[off] == '\\' {
146 if off+1 >= len(rest) || rest[off+1] == '\n' {
147 bad = true
148 } else {
149 off++
150 }
151 }
152 off++
153 }
154 text := strings.TrimSpace(rest[start:off])
155 if off < len(rest) {
156 off++
157 } else {
158 bad = true
159 }
160 s += take(off)
161 kind := "url"
162 if bad {
163 kind = "bad-url"
164 }
165 emit(kind, s, text, l, c)
166 continue
167 }
168 }
169 emit(k, s, value, l, c)
170 continue
171 }
172 s := take(n)
173 emit(s, s, s, l, c)
174 }
175 return out
176}
177func identStart(s string) bool {
178 if len(s) == 0 {
179 return false
180 }
181 r, _ := utf8.DecodeRuneInString(s)
182 return unicode.IsLetter(r) || r == '_' || r == '-' || r == '\\' || r >= 128
183}
184func identLen(s string) int {
185 i := 0
186 for i < len(s) {
187 r, n := utf8.DecodeRuneInString(s[i:])
188 if r == '\\' {
189 i += n
190 if i < len(s) {
191 _, e := escape(s[i:])
192 i += e
193 }
194 continue
195 }
196 if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '_' && r != '-' && r < 128 {
197 break
198 }
199 i += n
200 }
201 return i
202}
203func escape(s string) (rune, int) {
204 if len(s) == 0 {
205 return '\uFFFD', 0
206 }
207 i := 0
208 var x rune
209 for i < len(s) && i < 6 {
210 ch := s[i]
211 var d byte
212 switch {
213 case ch >= '0' && ch <= '9':
214 d = ch - '0'
215 case ch >= 'a' && ch <= 'f':
216 d = ch - 'a' + 10
217 case ch >= 'A' && ch <= 'F':
218 d = ch - 'A' + 10
219 default:
220 goto done
221 }
222 x = x*16 + rune(d)
223 i++
224 }
225done:
226 if i > 0 {
227 if i < len(s) && s[i] == ' ' {
228 i++
229 }
230 if x == 0 || x > utf8.MaxRune || x >= 0xd800 && x <= 0xdfff {
231 x = '\uFFFD'
232 }
233 return x, i
234 }
235 r, n := utf8.DecodeRuneInString(s)
236 return r, n
237}
238func numberStart(s string) bool {
239 if len(s) == 0 {
240 return false
241 }
242 i := 0
243 if s[i] == '+' || s[i] == '-' {
244 i++
245 }
246 if i < len(s) && s[i] >= '0' && s[i] <= '9' {
247 return true
248 }
249 return i+1 < len(s) && s[i] == '.' && s[i+1] >= '0' && s[i+1] <= '9'
250}
251func numberLen(s string) int {
252 i := 0
253 if s[i] == '+' || s[i] == '-' {
254 i++
255 }
256 for i < len(s) && s[i] >= '0' && s[i] <= '9' {
257 i++
258 }
259 if i < len(s) && s[i] == '.' {
260 i++
261 for i < len(s) && s[i] >= '0' && s[i] <= '9' {
262 i++
263 }
264 }
265 if i < len(s) && (s[i] == 'e' || s[i] == 'E') {
266 j := i + 1
267 if j < len(s) && (s[j] == '+' || s[j] == '-') {
268 j++
269 }
270 k := j
271 for j < len(s) && s[j] >= '0' && s[j] <= '9' {
272 j++
273 }
274 if j > k {
275 i = j
276 }
277 }
278 return i
279}
280
281func unescapeIdent(s string) string {
282 var b strings.Builder
283 for i := 0; i < len(s); {
284 if s[i] == '\\' && i+1 < len(s) {
285 r, n := escape(s[i+1:])
286 b.WriteRune(r)
287 i += n + 1
288 } else {
289 r, n := utf8.DecodeRuneInString(s[i:])
290 b.WriteRune(r)
291 i += n
292 }
293 }
294 return b.String()
295}