Package cssutils :: Package tests :: Module test_tokenize2
[hide private]
[frames] | no frames]

Source Code for Module cssutils.tests.test_tokenize2

  1  # -*- coding: utf-8 -*- 
  2  """testcases for new cssutils.tokenize.Tokenizer 
  3   
  4  TODO:: 
  5   
  6      - escape ends with explicit space but \r\n as single space 
  7      - ur'"\""': [('STRING', ur'"\""', 1, 1)], 
  8      - font-face with escaped "-" 
  9   
 10  + old tests as new ones are **not complete**! 
 11  """ 
 12  __author__ = '$LastChangedBy: cthedot $' 
 13  __date__ = '$LastChangedDate: 2007-09-01 15:56:36 +0200 (Sa, 01 Sep 2007) $' 
 14  __version__ = '$LastChangedRevision: 302 $' 
 15   
 16  import xml.dom 
 17  import basetest 
 18  from cssutils.tokenize2 import * 
 19   
20 -class TokenizerTestCase(basetest.BaseTestCase):
21 22 testsall = { 23 # IDENT 24 u'äöü߀': [('IDENT', u'äöü߀', 1, 1)], 25 u' a ': [('S', u' ', 1, 1), 26 ('IDENT', u'a', 1, 2), 27 ('S', u' ', 1, 3)], 28 u'_a': [('IDENT', u'_a', 1, 1)], 29 u'-a': [('IDENT', u'-a', 1, 1)], 30 u'aA-_\200\377': [('IDENT', u'aA-_\200\377', 1, 1)], 31 u'a1': [('IDENT', u'a1', 1, 1)], 32 # escapes must end with S or max 6 digits: 33 u'\\44 b': [('IDENT', u'Db', 1, 1)], 34 u'\\44 b': [('IDENT', u'D', 1, 1), 35 ('S', u' ', 1, 5), 36 ('IDENT', u'b', 1, 6)], 37 u'\\44\nb': [('IDENT', u'Db', 1, 1)], 38 u'\\44\rb': [('IDENT', u'Db', 1, 1)], 39 u'\\44\fb': [('IDENT', u'Db', 1, 1)], 40 u'\\44\n*': [('IDENT', u'D', 1, 1), 41 ('CHAR', u'*', 2, 1)], 42 u'\\44 a': [('IDENT', u'D', 1, 1), 43 ('S', u' ', 1, 5), 44 ('IDENT', u'a', 1, 6)], 45 # TODO: 46 # Note that this means that a "real" space after the escape sequence 47 # must itself either be escaped or doubled: 48 u'\\44\ x': [('IDENT', u'D\\ x', 1, 1)], 49 u'\\44 ': [('IDENT', u'D', 1, 1), 50 ('S', u' ', 1, 5)], 51 52 ur'\44': [('IDENT', u'D', 1, 1)], 53 ur'\\': [('IDENT', ur'\\', 1, 1)], 54 ur'\{': [('IDENT', ur'\{', 1, 1)], 55 ur'\"': [('IDENT', ur'\"', 1, 1)], 56 ur'\(': [('IDENT', ur'\(', 1, 1)], 57 ur'\1 \22 \333 \4444 \55555 \666666 \777777 7 \7777777': [ 58 ('IDENT', u'\x01"\u0333\u4444\\55555 \\666666 \\777777 7', 1, 1), 59 ('S', ' ', 1, 43), 60 ('IDENT', '\\7777777', 1, 44)], 61 62 63 u'\\1 b': [('IDENT', u'\x01b', 1, 1)], 64 u'\\44 b': [('IDENT', u'Db', 1, 1)], 65 u'\\123 b': [('IDENT', u'\u0123b', 1, 1)], 66 u'\\1234 b': [('IDENT', u'\u1234b', 1, 1)], 67 u'\\12345 b': [('IDENT', u'\\12345 b', 1, 1)], 68 u'\\123456 b': [('IDENT', u'\\123456 b', 1, 1)], 69 u'\\1234567 b': [('IDENT', u'\\1234567', 1, 1), 70 ('S', u' ', 1, 9), 71 ('IDENT', u'b', 1, 10)], 72 u'\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,': 73 [('IDENT', u'\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,', 1, 1)], 74 75 # STRING 76 u' "" ': [('S', u' ', 1, 1), 77 ('STRING', u'""', 1, 2), 78 ('S', u' ', 1, 4)], 79 u' "\'" ': [('S', u' ', 1, 1), 80 ('STRING', u'"\'"', 1, 2), 81 ('S', u' ', 1, 5)], 82 u" '' ": [('S', u' ', 1, 1), 83 ('STRING', u"''", 1, 2), 84 ('S', u' ', 1, 4)], 85 u" '' ": [('S', u' ', 1, 1), 86 ('STRING', u"''", 1, 2), 87 ('S', u' ', 1, 4)], 88 u"'\\\n'": [('STRING', u"'\\\n'", 1, 1)], 89 u"'\\\n\\\n\\\n'": [('STRING', u"'\\\n\\\n\\\n'", 1, 1)], 90 u"'\\\f'": [('STRING', u"'\\\f'", 1, 1)], 91 u"'\\\r'": [('STRING', u"'\\\r'", 1, 1)], 92 u"'\\\r\n'": [('STRING', u"'\\\r\n'", 1, 1)], 93 u"'1\\\n2'": [('STRING', u"'1\\\n2'", 1, 1)], 94 95 # HASH 96 u' #a ': [('S', u' ', 1, 1), 97 ('HASH', u'#a', 1, 2), 98 ('S', u' ', 1, 4)], 99 100 u'#ccc': [('HASH', u'#ccc', 1, 1)], 101 u'#111': [('HASH', u'#111', 1, 1)], 102 u'#a1a1a1': [('HASH', u'#a1a1a1', 1, 1)], 103 u'#1a1a1a': [('HASH', u'#1a1a1a', 1, 1)], 104 105 # NUMBER, for plus see CSS3 106 u' 0 ': [('S', u' ', 1, 1), 107 ('NUMBER', u'0', 1, 2), 108 ('S', u' ', 1, 3)], 109 u' 0.1 ': [('S', u' ', 1, 1), 110 ('NUMBER', u'0.1', 1, 2), 111 ('S', u' ', 1, 5)], 112 u' .0 ': [('S', u' ', 1, 1), 113 ('NUMBER', u'.0', 1, 2), 114 ('S', u' ', 1, 4)], 115 116 u' -0 ': [('S', u' ', 1, 1), 117 ('CHAR', u'-', 1, 2), 118 ('NUMBER', u'0', 1, 3), 119 ('S', u' ', 1, 4)], 120 121 # PERCENTAGE 122 u' 0% ': [('S', u' ', 1, 1), 123 ('PERCENTAGE', u'0%', 1, 2), 124 ('S', u' ', 1, 4)], 125 u' .5% ': [('S', u' ', 1, 1), 126 ('PERCENTAGE', u'.5%', 1, 2), 127 ('S', u' ', 1, 5)], 128 129 # URI 130 u' url() ': [('S', u' ', 1, 1), 131 ('URI', u'url()', 1, 2), 132 ('S', u' ', 1, 7)], 133 u' url(a) ': [('S', u' ', 1, 1), 134 ('URI', u'url(a)', 1, 2), 135 ('S', u' ', 1, 8)], 136 u' url("a") ': [('S', u' ', 1, 1), 137 ('URI', u'url("a")', 1, 2), 138 ('S', u' ', 1, 10)], 139 u' url( a ) ': [('S', u' ', 1, 1), 140 ('URI', u'url( a )', 1, 2), 141 ('S', u' ', 1, 10)], 142 143 # UNICODE-RANGE 144 145 # CDO 146 u' <!-- ': [('S', u' ', 1, 1), 147 ('CDO', u'<!--', 1, 2), 148 ('S', u' ', 1, 6)], 149 u'"<!--""-->"': [('STRING', u'"<!--"', 1, 1), 150 ('STRING', u'"-->"', 1, 7)], 151 152 # CDC 153 u' --> ': [('S', u' ', 1, 1), 154 ('CDC', u'-->', 1, 2), 155 ('S', u' ', 1, 5)], 156 157 # S 158 u' ': [('S', u' ', 1, 1)], 159 u' ': [('S', u' ', 1, 1)], 160 u'\r': [('S', u'\r', 1, 1)], 161 u'\n': [('S', u'\n', 1, 1)], 162 u'\r\n': [('S', u'\r\n', 1, 1)], 163 u'\f': [('S', u'\f', 1, 1)], 164 u'\r': [('S', u'\r', 1, 1)], 165 u'\t': [('S', u'\t', 1, 1)], 166 u'\r\n\r\n\f\t ': [('S', u'\r\n\r\n\f\t ', 1, 1)], 167 168 # COMMENT, for incomplete see later 169 u'/*x*/ ': [('COMMENT', u'/*x*/', 1, 1), 170 ('S', u' ', 1, 6)], 171 172 # FUNCTION 173 u' x( ': [('S', u' ', 1, 1), 174 ('FUNCTION', u'x(', 1, 2), 175 ('S', u' ', 1, 4)], 176 177 # INCLUDES 178 u' ~= ': [('S', u' ', 1, 1), 179 ('INCLUDES', u'~=', 1, 2), 180 ('S', u' ', 1, 4)], 181 u'~==': [('INCLUDES', u'~=', 1, 1), ('CHAR', u'=', 1, 3)], 182 183 # DASHMATCH 184 u' |= ': [('S', u' ', 1, 1), 185 ('DASHMATCH', u'|=', 1, 2), 186 ('S', u' ', 1, 4)], 187 u'|==': [('DASHMATCH', u'|=', 1, 1), ('CHAR', u'=', 1, 3)], 188 189 # CHAR 190 u' @ ': [('S', u' ', 1, 1), 191 ('CHAR', u'@', 1, 2), 192 ('S', u' ', 1, 3)], 193 194 # --- overwritten for CSS 2.1 --- 195 # LBRACE 196 u' { ': [('S', u' ', 1, 1), 197 ('CHAR', u'{', 1, 2), 198 ('S', u' ', 1, 3)], 199 # PLUS 200 u' + ': [('S', u' ', 1, 1), 201 ('CHAR', u'+', 1, 2), 202 ('S', u' ', 1, 3)], 203 # GREATER 204 u' > ': [('S', u' ', 1, 1), 205 ('CHAR', u'>', 1, 2), 206 ('S', u' ', 1, 3)], 207 # COMMA 208 u' , ': [('S', u' ', 1, 1), 209 ('CHAR', u',', 1, 2), 210 ('S', u' ', 1, 3)], 211 212 # class 213 u' . ': [('S', u' ', 1, 1), 214 ('CHAR', u'.', 1, 2), 215 ('S', u' ', 1, 3)], 216 217 } 218 219 tests3 = { 220 # specials 221 u'c\\olor': [('IDENT', u'c\\olor', 1, 1)], 222 u'-1': [('CHAR', u'-', 1, 1), ('NUMBER', u'1', 1, 2)], 223 u'-1px': [('CHAR', u'-', 1, 1), ('DIMENSION', u'1px', 1, 2)], 224 225 # ATKEYWORD 226 u' @x ': [('S', u' ', 1, 1), 227 ('ATKEYWORD', u'@x', 1, 2), 228 ('S', u' ', 1, 4)], 229 u'@X': [('ATKEYWORD', u'@X', 1, 1)], 230 u'@\\x': [('ATKEYWORD', u'@\\x', 1, 1)], 231 # - 232 u'@1x': [('CHAR', u'@', 1, 1), 233 ('DIMENSION', u'1x', 1, 2)], 234 235 # DIMENSION 236 u' 0px ': [('S', u' ', 1, 1), 237 ('DIMENSION', u'0px', 1, 2), 238 ('S', u' ', 1, 5)], 239 u' 1s ': [('S', u' ', 1, 1), 240 ('DIMENSION', u'1s', 1, 2), 241 ('S', u' ', 1, 4)], 242 u'0.2EM': [('DIMENSION', u'0.2EM', 1, 1)], 243 u'1p\\x': [('DIMENSION', u'1p\\x', 1, 1)], 244 u'1PX': [('DIMENSION', u'1PX', 1, 1)], 245 246 # NUMBER 247 u' - 0 ': [('S', u' ', 1, 1), 248 ('CHAR', u'-', 1, 2), 249 ('S', u' ', 1, 3), 250 ('NUMBER', u'0', 1, 4), 251 ('S', u' ', 1, 5)], 252 u' + 0 ': [('S', u' ', 1, 1), 253 ('CHAR', u'+', 1, 2), 254 ('S', u' ', 1, 3), 255 ('NUMBER', u'0', 1, 4), 256 ('S', u' ', 1, 5)], 257 258 # PREFIXMATCH 259 u' ^= ': [('S', u' ', 1, 1), 260 ('PREFIXMATCH', u'^=', 1, 2), 261 ('S', u' ', 1, 4)], 262 u'^==': [('PREFIXMATCH', u'^=', 1, 1), ('CHAR', u'=', 1, 3)], 263 264 # SUFFIXMATCH 265 u' $= ': [('S', u' ', 1, 1), 266 ('SUFFIXMATCH', u'$=', 1, 2), 267 ('S', u' ', 1, 4)], 268 u'$==': [('SUFFIXMATCH', u'$=', 1, 1), ('CHAR', u'=', 1, 3)], 269 270 # SUBSTRINGMATCH 271 u' *= ': [('S', u' ', 1, 1), 272 ('SUBSTRINGMATCH', u'*=', 1, 2), 273 ('S', u' ', 1, 4)], 274 u'*==': [('SUBSTRINGMATCH', u'*=', 1, 1), ('CHAR', u'=', 1, 3)], 275 276 # BOM 277 u' \xFEFF ': [('S', u' ', 1, 1), 278 ('BOM', u'\xFEFF', 1, 2), # len=3 279 ('S', u' ', 1, 5)], 280 281 } 282 283 tests2 = { 284 # escapes work not for a-f! 285 # IMPORT_SYM 286 u' @import ': [('S', u' ', 1, 1), 287 ('IMPORT_SYM', u'@import', 1, 2), 288 ('S', u' ', 1, 9)], 289 u'@IMPORT': [('IMPORT_SYM', u'@IMPORT', 1, 1)], 290 ur'@\i\m\p\o\r\t': [('IMPORT_SYM', ur'@\i\m\p\o\r\t', 1, 1)], 291 ur'@\I\M\P\O\R\T': [('IMPORT_SYM', ur'@\I\M\P\O\R\T', 1, 1)], 292 ur'@\49 \04d\0050\0004f\000052\54': [('IMPORT_SYM', 293 ur'@\49 \04d\0050\0004f\000052\54', 294 1, 1)], 295 ur'@\69 \06d\0070\0006f\000072\74': [('IMPORT_SYM', 296 ur'@\69 \06d\0070\0006f\000072\74', 297 1, 1)], 298 299 # PAGE_SYM 300 u' @page ': [('S', u' ', 1, 1), 301 ('PAGE_SYM', u'@page', 1, 2), 302 ('S', u' ', 1, 7)], 303 u'@PAGE': [('PAGE_SYM', u'@PAGE', 1, 1)], 304 ur'@\pa\ge': [('PAGE_SYM', ur'@\pa\ge', 1, 1)], 305 ur'@\PA\GE': [('PAGE_SYM', ur'@\PA\GE', 1, 1)], 306 ur'@\50\41\47\45': [('PAGE_SYM', ur'@\50\41\47\45', 1, 1)], 307 ur'@\70\61\67\65': [('PAGE_SYM', ur'@\70\61\67\65', 1, 1)], 308 309 # MEDIA_SYM 310 u' @media ': [('S', u' ', 1, 1), 311 ('MEDIA_SYM', u'@media', 1, 2), 312 ('S', u' ', 1, 8)], 313 u'@MEDIA': [('MEDIA_SYM', u'@MEDIA', 1, 1)], 314 ur'@\med\ia': [('MEDIA_SYM', ur'@\med\ia', 1, 1)], 315 ur'@\MED\IA': [('MEDIA_SYM', ur'@\MED\IA', 1, 1)], 316 u'@\\4d\n\\45\r\\44\t\\49\r\n\\41\f': [('MEDIA_SYM', 317 u'@\\4d\n\\45\r\\44\t\\49\r\n\\41\f', 318 1, 1)], 319 u'@\\6d\n\\65\r\\64\t\\69\r\n\\61\f': [('MEDIA_SYM', 320 u'@\\6d\n\\65\r\\64\t\\69\r\n\\61\f', 321 1, 1)], 322 323 # FONT_FACE_SYM 324 u' @font-face ': [('S', u' ', 1, 1), 325 ('FONT_FACE_SYM', u'@font-face', 1, 2), 326 ('S', u' ', 1, 12)], 327 u'@FONT-FACE': [('FONT_FACE_SYM', u'@FONT-FACE', 1, 1)], 328 ur'@f\o\n\t\-face': [('FONT_FACE_SYM', ur'@f\o\n\t\-face', 1, 1)], 329 ur'@F\O\N\T\-FACE': [('FONT_FACE_SYM', ur'@F\O\N\T\-FACE', 1, 1)], 330 # TODO: "-" as hex! 331 ur'@\46\4f\4e\54\-\46\41\43\45': [('FONT_FACE_SYM', 332 ur'@\46\4f\4e\54\-\46\41\43\45', 1, 1)], 333 ur'@\66\6f\6e\74\-\66\61\63\65': [('FONT_FACE_SYM', 334 ur'@\66\6f\6e\74\-\66\61\63\65', 1, 1)], 335 336 # CHARSET_SYM only if "@charset "! 337 u' @charset ': [('S', u' ', 1, 1), 338 ('CHARSET_SYM', u'@charset ', 1, 2), 339 ('S', u' ', 1, 11)], 340 u'@charset': [('ATKEYWORD', u'@charset', 1, 1)], # no ending S 341 u'@CHARSET ': [('ATKEYWORD', u'@CHARSET', 1, 1),# uppercase 342 ('S', u' ', 1, 9)], 343 u'@cha\\rset ': [('ATKEYWORD', u'@cha\\rset', 1, 1), # not literal 344 ('S', u' ', 1, 10)], 345 346 # NAMESPACE_SYM 347 u' @namespace ': [('S', u' ', 1, 1), 348 ('NAMESPACE_SYM', u'@namespace', 1, 2), 349 ('S', u' ', 1, 12)], 350 ur'@NAMESPACE': [('NAMESPACE_SYM', ur'@NAMESPACE', 1, 1)], 351 ur'@\na\me\s\pace': [('NAMESPACE_SYM', ur'@\na\me\s\pace', 1, 1)], 352 ur'@\NA\ME\S\PACE': [('NAMESPACE_SYM', ur'@\NA\ME\S\PACE', 1, 1)], 353 ur'@\4e\41\4d\45\53\50\41\43\45': [('NAMESPACE_SYM', 354 ur'@\4e\41\4d\45\53\50\41\43\45', 1, 1)], 355 ur'@\6e\61\6d\65\73\70\61\63\65': [('NAMESPACE_SYM', 356 ur'@\6e\61\6d\65\73\70\61\63\65', 1, 1)], 357 358 # ATKEYWORD 359 u' @unknown ': [('S', u' ', 1, 1), 360 ('ATKEYWORD', u'@unknown', 1, 2), 361 ('S', u' ', 1, 10)], 362 363 # STRING 364 # strings with linebreak in it 365 u' "\\na"\na': [('S', u' ', 1, 1), 366 ('STRING', u'"\\na"', 1, 2), 367 ('S', u'\n', 1, 7), 368 ('IDENT', u'a', 2, 1)], 369 u" '\\na'\na": [('S', u' ', 1, 1), 370 ('STRING', u"'\\na'", 1, 2), 371 ('S', u'\n', 1, 7), 372 ('IDENT', u'a', 2, 1)], 373 u' "\\r\\n\\t\\n\\ra"a': [('S', u' ', 1, 1), 374 ('STRING', u'"\\r\\n\\t\\n\\ra"', 1, 2), 375 ('IDENT', u'a', 1, 15)], 376 377 # IMPORTANT_SYM is not IDENT!!! 378 u' !important ': [('S', u' ', 1, 1), 379 ('CHAR', u'!', 1, 2), 380 ('IDENT', u'important', 1, 3), 381 ('S', u' ', 1, 12)], 382 u'! /*1*/ important ': [ 383 ('CHAR', u'!', 1, 1), 384 ('S', u' ', 1, 2), 385 ('COMMENT', u'/*1*/', 1, 3), 386 ('S', u' ', 1, 8), 387 ('IDENT', u'important', 1, 9), 388 ('S', u' ', 1, 18)], 389 u'! important': [('CHAR', u'!', 1, 1), 390 ('S', u' ', 1, 2), 391 ('IDENT', u'important', 1, 3)], 392 u'!\n\timportant': [('CHAR', u'!', 1, 1), 393 ('S', u'\n\t', 1, 2), 394 ('IDENT', u'important', 2, 2)], 395 u'!IMPORTANT': [('CHAR', u'!', 1, 1), 396 ('IDENT', u'IMPORTANT', 1, 2)], 397 ur'!\i\m\p\o\r\ta\n\t': [('CHAR', u'!', 1, 1), 398 ('IDENT', 399 ur'\i\m\p\o\r\ta\n\t', 1, 2)], 400 ur'!\I\M\P\O\R\Ta\N\T': [('CHAR', u'!', 1, 1), 401 ('IDENT', 402 ur'\I\M\P\O\R\Ta\N\T', 1, 2)], 403 ur'!\49\4d\50\4f\52\54\41\4e\54': [('CHAR', u'!', 1, 1), 404 ('IDENT', 405 ur'IMPORTANT', 406 1, 2)], 407 ur'!\69\6d\70\6f\72\74\61\6e\74': [('CHAR', u'!', 1, 1), 408 ('IDENT', 409 ur'important', 410 1, 2)], 411 } 412 413 # overwriting tests in testsall 414 tests2only = { 415 # LBRACE 416 u' { ': [('S', u' ', 1, 1), 417 ('LBRACE', u'{', 1, 2), 418 ('S', u' ', 1, 3)], 419 # PLUS 420 u' + ': [('S', u' ', 1, 1), 421 ('PLUS', u'+', 1, 2), 422 ('S', u' ', 1, 3)], 423 # GREATER 424 u' > ': [('S', u' ', 1, 1), 425 ('GREATER', u'>', 1, 2), 426 ('S', u' ', 1, 3)], 427 # COMMA 428 u' , ': [('S', u' ', 1, 1), 429 ('COMMA', u',', 1, 2), 430 ('S', u' ', 1, 3)], 431 # class 432 u' . ': [('S', u' ', 1, 1), 433 ('CLASS', u'.', 1, 2), 434 ('S', u' ', 1, 3)], 435 } 436 437 testsfullsheet = { 438 # TODO: escape ends with explicit space but \r\n as single space 439 #u'\\1\r\nb': [('IDENT', u'\\1\r', 1, 1), ('IDENT', u'b', 1, 4)], 440 441 # STRING 442 ur'"\" "': [('STRING', ur'"\" "', 1, 1)], 443 u"""'\\''""": [('STRING', u"""'\\''""", 1, 1)], 444 u'''"\\""''': [('STRING', u'''"\\""''', 1, 1)], 445 u' "\na': [('S', u' ', 1, 1), 446 ('INVALID', u'"', 1, 2), 447 ('S', u'\n', 1, 3), 448 ('IDENT', u'a', 2, 1)], 449 450 # strings with linebreak in it 451 u' "\\na\na': [('S', u' ', 1, 1), 452 ('INVALID', u'"\\na', 1, 2), 453 ('S', u'\n', 1, 6), 454 ('IDENT', u'a', 2, 1)], 455 u' "\\r\\n\\t\\n\\ra\na': [('S', u' ', 1, 1), 456 ('INVALID', u'"\\r\\n\\t\\n\\ra', 1, 2), 457 ('S', u'\n', 1, 14), 458 ('IDENT', u'a', 2, 1)], 459 # URI 460 u'ur\\l(a)': [('URI', u'ur\\l(a)', 1, 1)], 461 u'url(a)': [('URI', u'url(a)', 1, 1)], 462 u'\\55r\\4c(a)': [('URI', u'UrL(a)', 1, 1)], 463 u'\\75r\\6c(a)': [('URI', u'url(a)', 1, 1)], 464 } 465 466 # tests if fullsheet=False is set on tokenizer 467 testsfullsheetfalse = { 468 # COMMENT incomplete 469 u'/*': [('CHAR', u'/', 1, 1), 470 ('CHAR', u'*', 1, 2)], 471 472 # INVALID incomplete 473 u' " ': [('S', u' ', 1, 1), 474 ('INVALID', u'" ', 1, 2)], 475 u" 'abc\"with quote\" in it": [('S', u' ', 1, 1), 476 ('INVALID', u"'abc\"with quote\" in it", 1, 2)], 477 478 # URI incomplete 479 u'url(a': [('FUNCTION', u'url(', 1, 1), 480 ('IDENT', u'a', 1, 5)], 481 u'url("a': [('FUNCTION', u'url(', 1, 1), 482 ('INVALID', u'"a', 1, 5)], 483 u"url('a": [('FUNCTION', u'url(', 1, 1), 484 ('INVALID', u"'a", 1, 5)], 485 u"UR\\l('a": [('FUNCTION', u'UR\\l(', 1, 1), 486 ('INVALID', u"'a", 1, 6)], 487 } 488 489 # tests if fullsheet=True is set on tokenizer 490 testsfullsheettrue = { 491 # COMMENT incomplete 492 u'/*': [('COMMENT', u'/**/', 1, 1)], 493 494 # INVALID incomplete => STRING 495 u' " ': [('S', u' ', 1, 1), 496 ('STRING', u'" "', 1, 2)], 497 u" 'abc\"with quote\" in it": [('S', u' ', 1, 1), 498 ('STRING', u"'abc\"with quote\" in it'", 1, 2)], 499 500 # URI incomplete FUNC => URI 501 u'url(a': [('URI', u'url(a)', 1, 1)], 502 u'url( a': [('URI', u'url( a)', 1, 1)], 503 u'url("a': [('URI', u'url("a")', 1, 1)], 504 u'url( "a ': [('URI', u'url( "a ")', 1, 1)], 505 u"url('a": [('URI', u"url('a')", 1, 1)], 506 u'url("a"': [('URI', u'url("a")', 1, 1)], 507 u"url('a'": [('URI', u"url('a')", 1, 1)], 508 509 } 510
511 - def setUp(self):
512 #log = cssutils.errorhandler.ErrorHandler() 513 self.tokenizer = Tokenizer()
514
515 - def test_linenumbers(self):
516 "Tokenizer line + col" 517 pass
518
519 - def test_tokenize(self):
520 "cssutils Tokenizer().tokenize()" 521 import cssutils.cssproductions 522 tokenizer = Tokenizer(cssutils.cssproductions.MACROS, 523 cssutils.cssproductions.PRODUCTIONS) 524 tests = {} 525 tests.update(self.testsall) 526 tests.update(self.tests2) 527 tests.update(self.tests3) 528 tests.update(self.testsfullsheet) 529 tests.update(self.testsfullsheetfalse) 530 for css in tests: 531 # check token format 532 tokens = tokenizer.tokenize(css) 533 for i, actual in enumerate(tokens): 534 expected = tests[css][i] 535 self.assertEqual(expected, actual) 536 537 # check if all same number of tokens 538 tokens = [t for t in tokenizer.tokenize(css)] 539 self.assertEqual(len(tokens), len(tests[css]))
540
541 - def test_tokenizefullsheet(self):
542 "cssutils Tokenizer().tokenize(fullsheet=True)" 543 import cssutils.cssproductions 544 tokenizer = Tokenizer(cssutils.cssproductions.MACROS, 545 cssutils.cssproductions.PRODUCTIONS) 546 tests = {} 547 tests.update(self.testsall) 548 tests.update(self.tests2) 549 tests.update(self.tests3) 550 tests.update(self.testsfullsheet) 551 tests.update(self.testsfullsheettrue) 552 for css in tests: 553 # check token format 554 tokens = tokenizer.tokenize(css, fullsheet=True) 555 for i, actual in enumerate(tokens): 556 try: 557 expected = tests[css][i] 558 except IndexError: 559 # EOF is added 560 self.assertEqual(actual[0], 'EOF') 561 else: 562 self.assertEqual(expected, actual) 563 564 # check if all same number of tokens 565 tokens = [t for t in tokenizer.tokenize(css, fullsheet=True)] 566 # EOF is added so -1 567 self.assertEqual(len(tokens) - 1, len(tests[css]))
568 569 570 # not really needed
571 - def test_tokenizeCSS3(self):
572 "CSS3 Tokenizer().tokenize()" 573 import cssutils.css3productions 574 tokenizer = Tokenizer(cssutils.css3productions.MACROS, 575 cssutils.css3productions.PRODUCTIONS) 576 tests = {} 577 tests.update(self.testsall) 578 tests.update(self.tests3) 579 for css in tests: 580 tokens = tokenizer.tokenize(css) 581 for i, actual in enumerate(tokens): 582 expected = tests[css][i] 583 self.assertEqual(expected, actual)
584 585 # not really needed
586 - def test_tokenizeCSS2_1(self):
587 "CSS2 Tokenizer().tokenize()" 588 import cssutils.css2productions 589 tokenizer = Tokenizer(cssutils.css2productions.MACROS, 590 cssutils.css2productions.PRODUCTIONS) 591 tests = {} 592 tests.update(self.testsall) 593 #tests.update(self.tests2) 594 tests.update(self.tests2only) 595 for css in tests: 596 tokens = tokenizer.tokenize(css) 597 for i, actual in enumerate(tokens): 598 expected = tests[css][i] 599 self.assertEqual(expected, actual)
600 601 # -------------- 602
603 - def __old(self):
604 605 testsOLD = { 606 u'x x1 -x .-x #_x -': [(1, 1, tt.IDENT, u'x'), 607 (1, 2, 'S', u' '), 608 (1, 3, tt.IDENT, u'x1'), 609 (1, 5, 'S', u' '), 610 (1, 6, tt.IDENT, u'-x'), 611 (1, 8, 'S', u' '), 612 (1, 9, tt.CLASS, u'.'), 613 (1, 10, tt.IDENT, u'-x'), 614 (1, 12, 'S', u' '), 615 (1, 13, tt.HASH, u'#_x'), 616 (1, 16, 'S', u' '), 617 (1, 17, 'DELIM', u'-')], 618 619 # num 620 u'1 1.1 -1 -1.1 .1 -.1 1.': [(1, 1, tt.NUMBER, u'1'), 621 (1, 2, 'S', u' '), (1, 3, tt.NUMBER, u'1.1'), 622 (1, 6, 'S', u' '), (1, 7, tt.NUMBER, u'-1'), 623 (1, 9, 'S', u' '), (1, 10, tt.NUMBER, u'-1.1'), 624 (1, 14, 'S', u' '), (1, 15, tt.NUMBER, u'0.1'), 625 (1, 17, 'S', u' '), (1, 18, tt.NUMBER, u'-0.1'), 626 (1, 21, 'S', u' '), 627 (1, 22, tt.NUMBER, u'1'), (1, 23, tt.CLASS, u'.') 628 ], 629 # CSS3 pseudo 630 u'::': [(1, 1, tt.PSEUDO_ELEMENT, u'::')], 631 632 # SPECIALS 633 u'*+>~{},': [(1, 1, tt.UNIVERSAL, u'*'), 634 (1, 2, tt.PLUS, u'+'), 635 (1, 3, tt.GREATER, u'>'), 636 (1, 4, tt.TILDE, u'~'), 637 (1, 5, tt.LBRACE, u'{'), 638 (1, 6, tt.RBRACE, u'}'), 639 (1, 7, tt.COMMA, u',')], 640 641 # DELIM 642 u'!%:&$|': [(1, 1, 'DELIM', u'!'), 643 (1, 2, 'DELIM', u'%'), 644 (1, 3, 'DELIM', u':'), 645 (1, 4, 'DELIM', u'&'), 646 (1, 5, 'DELIM', u'$'), 647 (1, 6, 'DELIM', u'|')], 648 649 650 # DIMENSION 651 u'5em': [(1, 1, tt.DIMENSION, u'5em')], 652 u' 5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'5em')], 653 u'5em ': [(1, 1, tt.DIMENSION, u'5em'), (1, 4, 'S', u' ')], 654 655 u'-5em': [(1, 1, tt.DIMENSION, u'-5em')], 656 u' -5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'-5em')], 657 u'-5em ': [(1, 1, tt.DIMENSION, u'-5em'), (1, 5, 'S', u' ')], 658 659 u'.5em': [(1, 1, tt.DIMENSION, u'0.5em')], 660 u' .5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'0.5em')], 661 u'.5em ': [(1, 1, tt.DIMENSION, u'0.5em'), (1, 5, 'S', u' ')], 662 663 u'-.5em': [(1, 1, tt.DIMENSION, u'-0.5em')], 664 u' -.5em': [(1, 1, 'S', u' '), (1, 2, tt.DIMENSION, u'-0.5em')], 665 u'-.5em ': [(1, 1, tt.DIMENSION, u'-0.5em'), (1, 6, 'S', u' ')], 666 667 u'5em5_-': [(1, 1, tt.DIMENSION, u'5em5_-')], 668 669 u'a a5 a5a 5 5a 5a5': [(1, 1, tt.IDENT, u'a'), 670 (1, 2, 'S', u' '), 671 (1, 3, tt.IDENT, u'a5'), 672 (1, 5, 'S', u' '), 673 (1, 6, tt.IDENT, u'a5a'), 674 (1, 9, 'S', u' '), 675 (1, 10, tt.NUMBER, u'5'), 676 (1, 11, 'S', u' '), 677 (1, 12, tt.DIMENSION, u'5a'), 678 (1, 14, 'S', u' '), 679 (1, 15, tt.DIMENSION, u'5a5')], 680 681 # URI 682 u'url()': [(1, 1, tt.URI, u'url()')], 683 u'url();': [(1, 1, tt.URI, u'url()'), (1, 6, tt.SEMICOLON, ';')], 684 u'url("x")': [(1, 1, tt.URI, u'url("x")')], 685 u'url( "x")': [(1, 1, tt.URI, u'url("x")')], 686 u'url("x" )': [(1, 1, tt.URI, u'url("x")')], 687 u'url( "x" )': [(1, 1, tt.URI, u'url("x")')], 688 u' url("x")': [ 689 (1, 1, 'S', u' '), 690 (1, 2, tt.URI, u'url("x")')], 691 u'url("x") ': [ 692 (1, 1, tt.URI, u'url("x")'), 693 (1, 9, 'S', u' '), 694 ], 695 u'url(ab)': [(1, 1, tt.URI, u'url(ab)')], 696 u'url($#/ab)': [(1, 1, tt.URI, u'url($#/ab)')], 697 u'url(\1233/a/b)': [(1, 1, tt.URI, u'url(\1233/a/b)')], 698 # not URI 699 u'url("1""2")': [ 700 (1, 1, tt.FUNCTION, u'url('), 701 (1, 5, tt.STRING, u'"1"'), 702 (1, 8, tt.STRING, u'"2"'), 703 (1, 11, tt.RPARANTHESIS, u')'), 704 ], 705 u'url(a"2")': [ 706 (1, 1, tt.FUNCTION, u'url('), 707 (1, 5, tt.IDENT, u'a'), 708 (1, 6, tt.STRING, u'"2"'), 709 (1, 9, tt.RPARANTHESIS, u')'), 710 ], 711 u'url(a b)': [ 712 (1, 1, tt.FUNCTION, u'url('), 713 (1, 5, tt.IDENT, u'a'), 714 (1, 6, 'S', u' '), 715 (1, 7, tt.IDENT, u'b'), 716 (1, 8, tt.RPARANTHESIS, u')'), 717 ], 718 719 # FUNCTION 720 u' counter("x")': [ 721 (1,1, 'S', u' '), 722 (1, 2, tt.FUNCTION, u'counter('), 723 (1, 10, tt.STRING, u'"x"'), 724 (1, 13, tt.RPARANTHESIS, u')')], 725 # HASH 726 u'# #a #_a #-a #1': [ 727 (1, 1, 'DELIM', u'#'), 728 (1, 2, 'S', u' '), 729 (1, 3, tt.HASH, u'#a'), 730 (1, 5, 'S', u' '), 731 (1, 6, tt.HASH, u'#_a'), 732 (1, 9, 'S', u' '), 733 (1, 10, tt.HASH, u'#-a'), 734 (1, 13, 'S', u' '), 735 (1, 14, tt.HASH, u'#1') 736 ], 737 u'#1a1 ': [ 738 (1, 1, tt.HASH, u'#1a1'), 739 (1, 5, 'S', u' '), 740 ], 741 u'#1a1\n': [ 742 (1, 1, tt.HASH, u'#1a1'), 743 (1, 5, 'S', u'\n'), 744 ], 745 u'#1a1{': [ 746 (1, 1, tt.HASH, u'#1a1'), 747 (1, 5, tt.LBRACE, u'{'), 748 ], 749 u'#1a1 {': [ 750 (1, 1, tt.HASH, u'#1a1'), 751 (1, 5, 'S', u' '), 752 (1, 6, tt.LBRACE, u'{'), 753 ], 754 u'#1a1\n{': [ 755 (1, 1, tt.HASH, u'#1a1'), 756 (1, 5, 'S', u'\n'), 757 (2, 1, tt.LBRACE, u'{'), 758 ], 759 u'#1a1\n {': [ 760 (1, 1, tt.HASH, u'#1a1'), 761 (1, 5, 'S', u'\n '), 762 (2, 2, tt.LBRACE, u'{'), 763 ], 764 u'#1a1 \n{': [ 765 (1, 1, tt.HASH, u'#1a1'), 766 (1, 5, 'S', u' \n'), 767 (2, 1, tt.LBRACE, u'{'), 768 ], 769 # STRINGS with NL 770 u'"x\n': [(1,1, tt.INVALID, u'"x\n')], 771 u'"x\r': [(1,1, tt.INVALID, u'"x\r')], 772 u'"x\f': [(1,1, tt.INVALID, u'"x\f')], 773 u'"x\n ': [ 774 (1,1, tt.INVALID, u'"x\n'), 775 (2,1, 'S', u' ') 776 ] 777 778 }
779 780 # tests = { 781 # u'/*a': xml.dom.SyntaxErr, 782 # u'"a': xml.dom.SyntaxErr, 783 # u"'a": xml.dom.SyntaxErr, 784 # u"\\0 a": xml.dom.SyntaxErr, 785 # u"\\00": xml.dom.SyntaxErr, 786 # u"\\000": xml.dom.SyntaxErr, 787 # u"\\0000": xml.dom.SyntaxErr, 788 # u"\\00000": xml.dom.SyntaxErr, 789 # u"\\000000": xml.dom.SyntaxErr, 790 # u"\\0000001": xml.dom.SyntaxErr 791 # } 792 # self.tokenizer.log.raiseExceptions = True #!! 793 # for css, exception in tests.items(): 794 # self.assertRaises(exception, self.tokenizer.tokenize, css) 795 796 797 if __name__ == '__main__': 798 import unittest 799 unittest.main() 800