1
2 """testcases for new cssutils.tokenize.Tokenizer
3
4 TODO::
5
6 - escape ends with explicit space but \r\n as single space
7 - ur'"\""': [('STRING', ur'"\""', 1, 1)],
8 - font-face with escaped "-"
9
10 + old tests as new ones are **not complete**!
11 """
12 __author__ = '$LastChangedBy: cthedot $'
13 __date__ = '$LastChangedDate: 2007-09-01 15:56:36 +0200 (Sa, 01 Sep 2007) $'
14 __version__ = '$LastChangedRevision: 302 $'
15
16 import xml.dom
17 import basetest
18 from cssutils.tokenize2 import *
19
21
22 testsall = {
23
24 u'äöü߀': [('IDENT', u'äöü߀', 1, 1)],
25 u' a ': [('S', u' ', 1, 1),
26 ('IDENT', u'a', 1, 2),
27 ('S', u' ', 1, 3)],
28 u'_a': [('IDENT', u'_a', 1, 1)],
29 u'-a': [('IDENT', u'-a', 1, 1)],
30 u'aA-_\200\377': [('IDENT', u'aA-_\200\377', 1, 1)],
31 u'a1': [('IDENT', u'a1', 1, 1)],
32
33 u'\\44 b': [('IDENT', u'Db', 1, 1)],
34 u'\\44 b': [('IDENT', u'D', 1, 1),
35 ('S', u' ', 1, 5),
36 ('IDENT', u'b', 1, 6)],
37 u'\\44\nb': [('IDENT', u'Db', 1, 1)],
38 u'\\44\rb': [('IDENT', u'Db', 1, 1)],
39 u'\\44\fb': [('IDENT', u'Db', 1, 1)],
40 u'\\44\n*': [('IDENT', u'D', 1, 1),
41 ('CHAR', u'*', 2, 1)],
42 u'\\44 a': [('IDENT', u'D', 1, 1),
43 ('S', u' ', 1, 5),
44 ('IDENT', u'a', 1, 6)],
45
46
47
48 u'\\44\ x': [('IDENT', u'D\\ x', 1, 1)],
49 u'\\44 ': [('IDENT', u'D', 1, 1),
50 ('S', u' ', 1, 5)],
51
52 ur'\44': [('IDENT', u'D', 1, 1)],
53 ur'\\': [('IDENT', ur'\\', 1, 1)],
54 ur'\{': [('IDENT', ur'\{', 1, 1)],
55 ur'\"': [('IDENT', ur'\"', 1, 1)],
56 ur'\(': [('IDENT', ur'\(', 1, 1)],
57 ur'\1 \22 \333 \4444 \55555 \666666 \777777 7 \7777777': [
58 ('IDENT', u'\x01"\u0333\u4444\\55555 \\666666 \\777777 7', 1, 1),
59 ('S', ' ', 1, 43),
60 ('IDENT', '\\7777777', 1, 44)],
61
62
63 u'\\1 b': [('IDENT', u'\x01b', 1, 1)],
64 u'\\44 b': [('IDENT', u'Db', 1, 1)],
65 u'\\123 b': [('IDENT', u'\u0123b', 1, 1)],
66 u'\\1234 b': [('IDENT', u'\u1234b', 1, 1)],
67 u'\\12345 b': [('IDENT', u'\\12345 b', 1, 1)],
68 u'\\123456 b': [('IDENT', u'\\123456 b', 1, 1)],
69 u'\\1234567 b': [('IDENT', u'\\1234567', 1, 1),
70 ('S', u' ', 1, 9),
71 ('IDENT', u'b', 1, 10)],
72 u'\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,':
73 [('IDENT', u'\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,', 1, 1)],
74
75
76 u' "" ': [('S', u' ', 1, 1),
77 ('STRING', u'""', 1, 2),
78 ('S', u' ', 1, 4)],
79 u' "\'" ': [('S', u' ', 1, 1),
80 ('STRING', u'"\'"', 1, 2),
81 ('S', u' ', 1, 5)],
82 u" '' ": [('S', u' ', 1, 1),
83 ('STRING', u"''", 1, 2),
84 ('S', u' ', 1, 4)],
85 u" '' ": [('S', u' ', 1, 1),
86 ('STRING', u"''", 1, 2),
87 ('S', u' ', 1, 4)],
88 u"'\\\n'": [('STRING', u"'\\\n'", 1, 1)],
89 u"'\\\n\\\n\\\n'": [('STRING', u"'\\\n\\\n\\\n'", 1, 1)],
90 u"'\\\f'": [('STRING', u"'\\\f'", 1, 1)],
91 u"'\\\r'": [('STRING', u"'\\\r'", 1, 1)],
92 u"'\\\r\n'": [('STRING', u"'\\\r\n'", 1, 1)],
93 u"'1\\\n2'": [('STRING', u"'1\\\n2'", 1, 1)],
94
95
96 u' #a ': [('S', u' ', 1, 1),
97 ('HASH', u'#a', 1, 2),
98 ('S', u' ', 1, 4)],
99
100 u'#ccc': [('HASH', u'#ccc', 1, 1)],
101 u'#111': [('HASH', u'#111', 1, 1)],
102 u'#a1a1a1': [('HASH', u'#a1a1a1', 1, 1)],
103 u'#1a1a1a': [('HASH', u'#1a1a1a', 1, 1)],
104
105
106 u' 0 ': [('S', u' ', 1, 1),
107 ('NUMBER', u'0', 1, 2),
108 ('S', u' ', 1, 3)],
109 u' 0.1 ': [('S', u' ', 1, 1),
110 ('NUMBER', u'0.1', 1, 2),
111 ('S', u' ', 1, 5)],
112 u' .0 ': [('S', u' ', 1, 1),
113 ('NUMBER', u'.0', 1, 2),
114 ('S', u' ', 1, 4)],
115
116 u' -0 ': [('S', u' ', 1, 1),
117 ('CHAR', u'-', 1, 2),
118 ('NUMBER', u'0', 1, 3),
119 ('S', u' ', 1, 4)],
120
121
122 u' 0% ': [('S', u' ', 1, 1),
123 ('PERCENTAGE', u'0%', 1, 2),
124 ('S', u' ', 1, 4)],
125 u' .5% ': [('S', u' ', 1, 1),
126 ('PERCENTAGE', u'.5%', 1, 2),
127 ('S', u' ', 1, 5)],
128
129
130 u' url() ': [('S', u' ', 1, 1),
131 ('URI', u'url()', 1, 2),
132 ('S', u' ', 1, 7)],
133 u' url(a) ': [('S', u' ', 1, 1),
134 ('URI', u'url(a)', 1, 2),
135 ('S', u' ', 1, 8)],
136 u' url("a") ': [('S', u' ', 1, 1),
137 ('URI', u'url("a")', 1, 2),
138 ('S', u' ', 1, 10)],
139 u' url( a ) ': [('S', u' ', 1, 1),
140 ('URI', u'url( a )', 1, 2),
141 ('S', u' ', 1, 10)],
142
143
144
145
146 u' <!-- ': [('S', u' ', 1, 1),
147 ('CDO', u'<!--', 1, 2),
148 ('S', u' ', 1, 6)],
149 u'"<!--""-->"': [('STRING', u'"<!--"', 1, 1),
150 ('STRING', u'"-->"', 1, 7)],
151
152
153 u' --> ': [('S', u' ', 1, 1),
154 ('CDC', u'-->', 1, 2),
155 ('S', u' ', 1, 5)],
156
157
158 u' ': [('S', u' ', 1, 1)],
159 u' ': [('S', u' ', 1, 1)],
160 u'\r': [('S', u'\r', 1, 1)],
161 u'\n': [('S', u'\n', 1, 1)],
162 u'\r\n': [('S', u'\r\n', 1, 1)],
163 u'\f': [('S', u'\f', 1, 1)],
164 u'\r': [('S', u'\r', 1, 1)],
165 u'\t': [('S', u'\t', 1, 1)],
166 u'\r\n\r\n\f\t ': [('S', u'\r\n\r\n\f\t ', 1, 1)],
167
168
169 u'/*x*/ ': [('COMMENT', u'/*x*/', 1, 1),
170 ('S', u' ', 1, 6)],
171
172
173 u' x( ': [('S', u' ', 1, 1),
174 ('FUNCTION', u'x(', 1, 2),
175 ('S', u' ', 1, 4)],
176
177
178 u' ~= ': [('S', u' ', 1, 1),
179 ('INCLUDES', u'~=', 1, 2),
180 ('S', u' ', 1, 4)],
181 u'~==': [('INCLUDES', u'~=', 1, 1), ('CHAR', u'=', 1, 3)],
182
183
184 u' |= ': [('S', u' ', 1, 1),
185 ('DASHMATCH', u'|=', 1, 2),
186 ('S', u' ', 1, 4)],
187 u'|==': [('DASHMATCH', u'|=', 1, 1), ('CHAR', u'=', 1, 3)],
188
189
190 u' @ ': [('S', u' ', 1, 1),
191 ('CHAR', u'@', 1, 2),
192 ('S', u' ', 1, 3)],
193
194
195
196 u' { ': [('S', u' ', 1, 1),
197 ('CHAR', u'{', 1, 2),
198 ('S', u' ', 1, 3)],
199
200 u' + ': [('S', u' ', 1, 1),
201 ('CHAR', u'+', 1, 2),
202 ('S', u' ', 1, 3)],
203
204 u' > ': [('S', u' ', 1, 1),
205 ('CHAR', u'>', 1, 2),
206 ('S', u' ', 1, 3)],
207
208 u' , ': [('S', u' ', 1, 1),
209 ('CHAR', u',', 1, 2),
210 ('S', u' ', 1, 3)],
211
212
213 u' . ': [('S', u' ', 1, 1),
214 ('CHAR', u'.', 1, 2),
215 ('S', u' ', 1, 3)],
216
217 }
218
219 tests3 = {
220
221 u'c\\olor': [('IDENT', u'c\\olor', 1, 1)],
222 u'-1': [('CHAR', u'-', 1, 1), ('NUMBER', u'1', 1, 2)],
223 u'-1px': [('CHAR', u'-', 1, 1), ('DIMENSION', u'1px', 1, 2)],
224
225
226 u' @x ': [('S', u' ', 1, 1),
227 ('ATKEYWORD', u'@x', 1, 2),
228 ('S', u' ', 1, 4)],
229 u'@X': [('ATKEYWORD', u'@X', 1, 1)],
230 u'@\\x': [('ATKEYWORD', u'@\\x', 1, 1)],
231
232 u'@1x': [('CHAR', u'@', 1, 1),
233 ('DIMENSION', u'1x', 1, 2)],
234
235
236 u' 0px ': [('S', u' ', 1, 1),
237 ('DIMENSION', u'0px', 1, 2),
238 ('S', u' ', 1, 5)],
239 u' 1s ': [('S', u' ', 1, 1),
240 ('DIMENSION', u'1s', 1, 2),
241 ('S', u' ', 1, 4)],
242 u'0.2EM': [('DIMENSION', u'0.2EM', 1, 1)],
243 u'1p\\x': [('DIMENSION', u'1p\\x', 1, 1)],
244 u'1PX': [('DIMENSION', u'1PX', 1, 1)],
245
246
247 u' - 0 ': [('S', u' ', 1, 1),
248 ('CHAR', u'-', 1, 2),
249 ('S', u' ', 1, 3),
250 ('NUMBER', u'0', 1, 4),
251 ('S', u' ', 1, 5)],
252 u' + 0 ': [('S', u' ', 1, 1),
253 ('CHAR', u'+', 1, 2),
254 ('S', u' ', 1, 3),
255 ('NUMBER', u'0', 1, 4),
256 ('S', u' ', 1, 5)],
257
258
259 u' ^= ': [('S', u' ', 1, 1),
260 ('PREFIXMATCH', u'^=', 1, 2),
261 ('S', u' ', 1, 4)],
262 u'^==': [('PREFIXMATCH', u'^=', 1, 1), ('CHAR', u'=', 1, 3)],
263
264
265 u' $= ': [('S', u' ', 1, 1),
266 ('SUFFIXMATCH', u'$=', 1, 2),
267 ('S', u' ', 1, 4)],
268 u'$==': [('SUFFIXMATCH', u'$=', 1, 1), ('CHAR', u'=', 1, 3)],
269
270
271 u' *= ': [('S', u' ', 1, 1),
272 ('SUBSTRINGMATCH', u'*=', 1, 2),
273 ('S', u' ', 1, 4)],
274 u'*==': [('SUBSTRINGMATCH', u'*=', 1, 1), ('CHAR', u'=', 1, 3)],
275
276
277 u' \xFEFF ': [('S', u' ', 1, 1),
278 ('BOM', u'\xFEFF', 1, 2),
279 ('S', u' ', 1, 5)],
280
281 }
282
283 tests2 = {
284
285
286 u' @import ': [('S', u' ', 1, 1),
287 ('IMPORT_SYM', u'@import', 1, 2),
288 ('S', u' ', 1, 9)],
289 u'@IMPORT': [('IMPORT_SYM', u'@IMPORT', 1, 1)],
290 ur'@\i\m\p\o\r\t': [('IMPORT_SYM', ur'@\i\m\p\o\r\t', 1, 1)],
291 ur'@\I\M\P\O\R\T': [('IMPORT_SYM', ur'@\I\M\P\O\R\T', 1, 1)],
292 ur'@\49 \04d\0050\0004f\000052\54': [('IMPORT_SYM',
293 ur'@\49 \04d\0050\0004f\000052\54',
294 1, 1)],
295 ur'@\69 \06d\0070\0006f\000072\74': [('IMPORT_SYM',
296 ur'@\69 \06d\0070\0006f\000072\74',
297 1, 1)],
298
299
300 u' @page ': [('S', u' ', 1, 1),
301 ('PAGE_SYM', u'@page', 1, 2),
302 ('S', u' ', 1, 7)],
303 u'@PAGE': [('PAGE_SYM', u'@PAGE', 1, 1)],
304 ur'@\pa\ge': [('PAGE_SYM', ur'@\pa\ge', 1, 1)],
305 ur'@\PA\GE': [('PAGE_SYM', ur'@\PA\GE', 1, 1)],
306 ur'@\50\41\47\45': [('PAGE_SYM', ur'@\50\41\47\45', 1, 1)],
307 ur'@\70\61\67\65': [('PAGE_SYM', ur'@\70\61\67\65', 1, 1)],
308
309
310 u' @media ': [('S', u' ', 1, 1),
311 ('MEDIA_SYM', u'@media', 1, 2),
312 ('S', u' ', 1, 8)],
313 u'@MEDIA': [('MEDIA_SYM', u'@MEDIA', 1, 1)],
314 ur'@\med\ia': [('MEDIA_SYM', ur'@\med\ia', 1, 1)],
315 ur'@\MED\IA': [('MEDIA_SYM', ur'@\MED\IA', 1, 1)],
316 u'@\\4d\n\\45\r\\44\t\\49\r\n\\41\f': [('MEDIA_SYM',
317 u'@\\4d\n\\45\r\\44\t\\49\r\n\\41\f',
318 1, 1)],
319 u'@\\6d\n\\65\r\\64\t\\69\r\n\\61\f': [('MEDIA_SYM',
320 u'@\\6d\n\\65\r\\64\t\\69\r\n\\61\f',
321 1, 1)],
322
323
324 u' @font-face ': [('S', u' ', 1, 1),
325 ('FONT_FACE_SYM', u'@font-face', 1, 2),
326 ('S', u' ', 1, 12)],
327 u'@FONT-FACE': [('FONT_FACE_SYM', u'@FONT-FACE', 1, 1)],
328 ur'@f\o\n\t\-face': [('FONT_FACE_SYM', ur'@f\o\n\t\-face', 1, 1)],
329 ur'@F\O\N\T\-FACE': [('FONT_FACE_SYM', ur'@F\O\N\T\-FACE', 1, 1)],
330
331 ur'@\46\4f\4e\54\-\46\41\43\45': [('FONT_FACE_SYM',
332 ur'@\46\4f\4e\54\-\46\41\43\45', 1, 1)],
333 ur'@\66\6f\6e\74\-\66\61\63\65': [('FONT_FACE_SYM',
334 ur'@\66\6f\6e\74\-\66\61\63\65', 1, 1)],
335
336
337 u' @charset ': [('S', u' ', 1, 1),
338 ('CHARSET_SYM', u'@charset ', 1, 2),
339 ('S', u' ', 1, 11)],
340 u'@charset': [('ATKEYWORD', u'@charset', 1, 1)],
341 u'@CHARSET ': [('ATKEYWORD', u'@CHARSET', 1, 1),
342 ('S', u' ', 1, 9)],
343 u'@cha\\rset ': [('ATKEYWORD', u'@cha\\rset', 1, 1),
344 ('S', u' ', 1, 10)],
345
346
347 u' @namespace ': [('S', u' ', 1, 1),
348 ('NAMESPACE_SYM', u'@namespace', 1, 2),
349 ('S', u' ', 1, 12)],
350 ur'@NAMESPACE': [('NAMESPACE_SYM', ur'@NAMESPACE', 1, 1)],
351 ur'@\na\me\s\pace': [('NAMESPACE_SYM', ur'@\na\me\s\pace', 1, 1)],
352 ur'@\NA\ME\S\PACE': [('NAMESPACE_SYM', ur'@\NA\ME\S\PACE', 1, 1)],
353 ur'@\4e\41\4d\45\53\50\41\43\45': [('NAMESPACE_SYM',
354 ur'@\4e\41\4d\45\53\50\41\43\45', 1, 1)],
355 ur'@\6e\61\6d\65\73\70\61\63\65': [('NAMESPACE_SYM',
356 ur'@\6e\61\6d\65\73\70\61\63\65', 1, 1)],
357
358
359 u' @unknown ': [('S', u' ', 1, 1),
360 ('ATKEYWORD', u'@unknown', 1, 2),
361 ('S', u' ', 1, 10)],
362
363
364
365 u' "\\na"\na': [('S', u' ', 1, 1),
366 ('STRING', u'"\\na"', 1, 2),
367 ('S', u'\n', 1, 7),
368 ('IDENT', u'a', 2, 1)],
369 u" '\\na'\na": [('S', u' ', 1, 1),
370 ('STRING', u"'\\na'", 1, 2),
371 ('S', u'\n', 1, 7),
372 ('IDENT', u'a', 2, 1)],
373 u' "\\r\\n\\t\\n\\ra"a': [('S', u' ', 1, 1),
374 ('STRING', u'"\\r\\n\\t\\n\\ra"', 1, 2),
375 ('IDENT', u'a', 1, 15)],
376
377
378 u' !important ': [('S', u' ', 1, 1),
379 ('CHAR', u'!', 1, 2),
380 ('IDENT', u'important', 1, 3),
381 ('S', u' ', 1, 12)],
382 u'! /*1*/ important ': [
383 ('CHAR', u'!', 1, 1),
384 ('S', u' ', 1, 2),
385 ('COMMENT', u'/*1*/', 1, 3),
386 ('S', u' ', 1, 8),
387 ('IDENT', u'important', 1, 9),
388 ('S', u' ', 1, 18)],
389 u'! important': [('CHAR', u'!', 1, 1),
390 ('S', u' ', 1, 2),
391 ('IDENT', u'important', 1, 3)],
392 u'!\n\timportant': [('CHAR', u'!', 1, 1),
393 ('S', u'\n\t', 1, 2),
394 ('IDENT', u'important', 2, 2)],
395 u'!IMPORTANT': [('CHAR', u'!', 1, 1),
396 ('IDENT', u'IMPORTANT', 1, 2)],
397 ur'!\i\m\p\o\r\ta\n\t': [('CHAR', u'!', 1, 1),
398 ('IDENT',
399 ur'\i\m\p\o\r\ta\n\t', 1, 2)],
400 ur'!\I\M\P\O\R\Ta\N\T': [('CHAR', u'!', 1, 1),
401 ('IDENT',
402 ur'\I\M\P\O\R\Ta\N\T', 1, 2)],
403 ur'!\49\4d\50\4f\52\54\41\4e\54': [('CHAR', u'!', 1, 1),
404 ('IDENT',
405 ur'IMPORTANT',
406 1, 2)],
407 ur'!\69\6d\70\6f\72\74\61\6e\74': [('CHAR', u'!', 1, 1),
408 ('IDENT',
409 ur'important',
410 1, 2)],
411 }
412
413
414 tests2only = {
415
416 u' { ': [('S', u' ', 1, 1),
417 ('LBRACE', u'{', 1, 2),
418 ('S', u' ', 1, 3)],
419
420 u' + ': [('S', u' ', 1, 1),
421 ('PLUS', u'+', 1, 2),
422 ('S', u' ', 1, 3)],
423
424 u' > ': [('S', u' ', 1, 1),
425 ('GREATER', u'>', 1, 2),
426 ('S', u' ', 1, 3)],
427
428 u' , ': [('S', u' ', 1, 1),
429 ('COMMA', u',', 1, 2),
430 ('S', u' ', 1, 3)],
431
432 u' . ': [('S', u' ', 1, 1),
433 ('CLASS', u'.', 1, 2),
434 ('S', u' ', 1, 3)],
435 }
436
437 testsfullsheet = {
438
439
440
441
442 ur'"\" "': [('STRING', ur'"\" "', 1, 1)],
443 u"""'\\''""": [('STRING', u"""'\\''""", 1, 1)],
444 u'''"\\""''': [('STRING', u'''"\\""''', 1, 1)],
445 u' "\na': [('S', u' ', 1, 1),
446 ('INVALID', u'"', 1, 2),
447 ('S', u'\n', 1, 3),
448 ('IDENT', u'a', 2, 1)],
449
450
451 u' "\\na\na': [('S', u' ', 1, 1),
452 ('INVALID', u'"\\na', 1, 2),
453 ('S', u'\n', 1, 6),
454 ('IDENT', u'a', 2, 1)],
455 u' "\\r\\n\\t\\n\\ra\na': [('S', u' ', 1, 1),
456 ('INVALID', u'"\\r\\n\\t\\n\\ra', 1, 2),
457 ('S', u'\n', 1, 14),
458 ('IDENT', u'a', 2, 1)],
459
460 u'ur\\l(a)': [('URI', u'ur\\l(a)', 1, 1)],
461 u'url(a)': [('URI', u'url(a)', 1, 1)],
462 u'\\55r\\4c(a)': [('URI', u'UrL(a)', 1, 1)],
463 u'\\75r\\6c(a)': [('URI', u'url(a)', 1, 1)],
464 }
465
466
467 testsfullsheetfalse = {
468
469 u'/*': [('CHAR', u'/', 1, 1),
470 ('CHAR', u'*', 1, 2)],
471
472
473 u' " ': [('S', u' ', 1, 1),
474 ('INVALID', u'" ', 1, 2)],
475 u" 'abc\"with quote\" in it": [('S', u' ', 1, 1),
476 ('INVALID', u"'abc\"with quote\" in it", 1, 2)],
477
478
479 u'url(a': [('FUNCTION', u'url(', 1, 1),
480 ('IDENT', u'a', 1, 5)],
481 u'url("a': [('FUNCTION', u'url(', 1, 1),
482 ('INVALID', u'"a', 1, 5)],
483 u"url('a": [('FUNCTION', u'url(', 1, 1),
484 ('INVALID', u"'a", 1, 5)],
485 u"UR\\l('a": [('FUNCTION', u'UR\\l(', 1, 1),
486 ('INVALID', u"'a", 1, 6)],
487 }
488
489
490 testsfullsheettrue = {
491
492 u'/*': [('COMMENT', u'/**/', 1, 1)],
493
494
495 u' " ': [('S', u' ', 1, 1),
496 ('STRING', u'" "', 1, 2)],
497 u" 'abc\"with quote\" in it": [('S', u' ', 1, 1),
498 ('STRING', u"'abc\"with quote\" in it'", 1, 2)],
499
500
501 u'url(a': [('URI', u'url(a)', 1, 1)],
502 u'url( a': [('URI', u'url( a)', 1, 1)],
503 u'url("a': [('URI', u'url("a")', 1, 1)],
504 u'url( "a ': [('URI', u'url( "a ")', 1, 1)],
505 u"url('a": [('URI', u"url('a')", 1, 1)],
506 u'url("a"': [('URI', u'url("a")', 1, 1)],
507 u"url('a'": [('URI', u"url('a')", 1, 1)],
508
509 }
510
514
516 "Tokenizer line + col"
517 pass
518
540
568
569
570
584
585
600
601
602
604
605 testsOLD = {
606 u'x x1 -x .-x #_x -': [(1, 1, tt.IDENT, u'x'),
607 (1, 2, 'S', u' '),
608 (1, 3, tt.IDENT, u'x1'),
609 (1, 5, 'S', u' '),
610 (1, 6, tt.IDENT, u'-x'),
611 (1, 8, 'S', u' '),
612 (1, 9, tt.CLASS, u'.'),
613 (1, 10, tt.IDENT, u'-x'),
614 (1, 12, 'S', u' '),
615 (1, 13, tt.HASH, u'#_x'),
616 (1, 16, 'S', u' '),
617 (1, 17, 'DELIM', u'-')],
618
619
620 u'1 1.1 -1 -1.1 .1 -.1 1.': [(1, 1, tt.NUMBER, u'1'),
621 (1, 2, 'S', u' '), (1, 3, tt.NUMBER, u'1.1'),
622 (1, 6, 'S', u' '), (1, 7, tt.NUMBER, u'-1'),
623 (1, 9, 'S', u' '), (1, 10, tt.NUMBER, u'-1.1'),
624 (1, 14, 'S', u' '), (1, 15, tt.NUMBER, u'0.1'),
625 (1, 17, 'S', u' '), (1, 18, tt.NUMBER, u'-0.1'),
626 (1, 21, 'S', u' '),
627 (1, 22, tt.NUMBER, u'1'), (1, 23, tt.CLASS, u'.')
628 ],
629
630 u'::': [(1, 1, tt.PSEUDO_ELEMENT, u'::')],
631
632
633 u'*+>~{},': [(1, 1, tt.UNIVERSAL, u'*'),
634 (1, 2, tt.PLUS, u'+'),
635 (1, 3, tt.GREATER, u'>'),
636 (1, 4, tt.TILDE, u'~'),
637 (1, 5, tt.LBRACE, u'{'),
638 (1, 6, tt.RBRACE, u'}'),
639 (1, 7, tt.COMMA, u',')],
640
641
642 u'!%:&$|': [(1, 1, 'DELIM', u'!'),
643 (1, 2, 'DELIM', u'%'),
644 (1, 3, 'DELIM', u':'),
645 (1, 4, 'DELIM', u'&'),
646 (1, 5, 'DELIM', u'$'),
647 (1, 6, 'DELIM', u'|')],
648
649
650
651 u'5em': [(1, 1, tt.DIMENSION, u'5em')],
652 u' 5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'5em')],
653 u'5em ': [(1, 1, tt.DIMENSION, u'5em'), (1, 4, 'S', u' ')],
654
655 u'-5em': [(1, 1, tt.DIMENSION, u'-5em')],
656 u' -5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'-5em')],
657 u'-5em ': [(1, 1, tt.DIMENSION, u'-5em'), (1, 5, 'S', u' ')],
658
659 u'.5em': [(1, 1, tt.DIMENSION, u'0.5em')],
660 u' .5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'0.5em')],
661 u'.5em ': [(1, 1, tt.DIMENSION, u'0.5em'), (1, 5, 'S', u' ')],
662
663 u'-.5em': [(1, 1, tt.DIMENSION, u'-0.5em')],
664 u' -.5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'-0.5em')],
665 u'-.5em ': [(1, 1, tt.DIMENSION, u'-0.5em'), (1, 6, 'S', u' ')],
666
667 u'5em5_-': [(1, 1, tt.DIMENSION, u'5em5_-')],
668
669 u'a a5 a5a 5 5a 5a5': [(1, 1, tt.IDENT, u'a'),
670 (1, 2, 'S', u' '),
671 (1, 3, tt.IDENT, u'a5'),
672 (1, 5, 'S', u' '),
673 (1, 6, tt.IDENT, u'a5a'),
674 (1, 9, 'S', u' '),
675 (1, 10, tt.NUMBER, u'5'),
676 (1, 11, 'S', u' '),
677 (1, 12, tt.DIMENSION, u'5a'),
678 (1, 14, 'S', u' '),
679 (1, 15, tt.DIMENSION, u'5a5')],
680
681
682 u'url()': [(1, 1, tt.URI, u'url()')],
683 u'url();': [(1, 1, tt.URI, u'url()'), (1, 6, tt.SEMICOLON, ';')],
684 u'url("x")': [(1, 1, tt.URI, u'url("x")')],
685 u'url( "x")': [(1, 1, tt.URI, u'url("x")')],
686 u'url("x" )': [(1, 1, tt.URI, u'url("x")')],
687 u'url( "x" )': [(1, 1, tt.URI, u'url("x")')],
688 u' url("x")': [
689 (1, 1, 'S', u' '),
690 (1, 2, tt.URI, u'url("x")')],
691 u'url("x") ': [
692 (1, 1, tt.URI, u'url("x")'),
693 (1, 9, 'S', u' '),
694 ],
695 u'url(ab)': [(1, 1, tt.URI, u'url(ab)')],
696 u'url($#/ab)': [(1, 1, tt.URI, u'url($#/ab)')],
697 u'url(\1233/a/b)': [(1, 1, tt.URI, u'url(\1233/a/b)')],
698
699 u'url("1""2")': [
700 (1, 1, tt.FUNCTION, u'url('),
701 (1, 5, tt.STRING, u'"1"'),
702 (1, 8, tt.STRING, u'"2"'),
703 (1, 11, tt.RPARANTHESIS, u')'),
704 ],
705 u'url(a"2")': [
706 (1, 1, tt.FUNCTION, u'url('),
707 (1, 5, tt.IDENT, u'a'),
708 (1, 6, tt.STRING, u'"2"'),
709 (1, 9, tt.RPARANTHESIS, u')'),
710 ],
711 u'url(a b)': [
712 (1, 1, tt.FUNCTION, u'url('),
713 (1, 5, tt.IDENT, u'a'),
714 (1, 6, 'S', u' '),
715 (1, 7, tt.IDENT, u'b'),
716 (1, 8, tt.RPARANTHESIS, u')'),
717 ],
718
719
720 u' counter("x")': [
721 (1,1, 'S', u' '),
722 (1, 2, tt.FUNCTION, u'counter('),
723 (1, 10, tt.STRING, u'"x"'),
724 (1, 13, tt.RPARANTHESIS, u')')],
725
726 u'# #a #_a #-a #1': [
727 (1, 1, 'DELIM', u'#'),
728 (1, 2, 'S', u' '),
729 (1, 3, tt.HASH, u'#a'),
730 (1, 5, 'S', u' '),
731 (1, 6, tt.HASH, u'#_a'),
732 (1, 9, 'S', u' '),
733 (1, 10, tt.HASH, u'#-a'),
734 (1, 13, 'S', u' '),
735 (1, 14, tt.HASH, u'#1')
736 ],
737 u'#1a1 ': [
738 (1, 1, tt.HASH, u'#1a1'),
739 (1, 5, 'S', u' '),
740 ],
741 u'#1a1\n': [
742 (1, 1, tt.HASH, u'#1a1'),
743 (1, 5, 'S', u'\n'),
744 ],
745 u'#1a1{': [
746 (1, 1, tt.HASH, u'#1a1'),
747 (1, 5, tt.LBRACE, u'{'),
748 ],
749 u'#1a1 {': [
750 (1, 1, tt.HASH, u'#1a1'),
751 (1, 5, 'S', u' '),
752 (1, 6, tt.LBRACE, u'{'),
753 ],
754 u'#1a1\n{': [
755 (1, 1, tt.HASH, u'#1a1'),
756 (1, 5, 'S', u'\n'),
757 (2, 1, tt.LBRACE, u'{'),
758 ],
759 u'#1a1\n {': [
760 (1, 1, tt.HASH, u'#1a1'),
761 (1, 5, 'S', u'\n '),
762 (2, 2, tt.LBRACE, u'{'),
763 ],
764 u'#1a1 \n{': [
765 (1, 1, tt.HASH, u'#1a1'),
766 (1, 5, 'S', u' \n'),
767 (2, 1, tt.LBRACE, u'{'),
768 ],
769
770 u'"x\n': [(1,1, tt.INVALID, u'"x\n')],
771 u'"x\r': [(1,1, tt.INVALID, u'"x\r')],
772 u'"x\f': [(1,1, tt.INVALID, u'"x\f')],
773 u'"x\n ': [
774 (1,1, tt.INVALID, u'"x\n'),
775 (2,1, 'S', u' ')
776 ]
777
778 }
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797 if __name__ == '__main__':
798 import unittest
799 unittest.main()
800