lexer.py (17286B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\lib\jinja2\lexer.py 2 import re 3 from operator import itemgetter 4 from collections import deque 5 from jinja2.exceptions import TemplateSyntaxError 6 from jinja2.utils import LRUCache, next 7 _lexer_cache = LRUCache(50) 8 whitespace_re = re.compile('\\s+', re.U) 9 string_re = re.compile('(\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'|"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)")', re.S) 10 integer_re = re.compile('\\d+') 11 try: 12 compile('f\xc3\xb6\xc3\xb6', '<unknown>', 'eval') 13 except SyntaxError: 14 name_re = re.compile('\\b[a-zA-Z_][a-zA-Z0-9_]*\\b') 15 else: 16 from jinja2 import _stringdefs 17 name_re = re.compile('[%s][%s]*' % (_stringdefs.xid_start, _stringdefs.xid_continue)) 18 19 float_re = re.compile('(?<!\\.)\\d+\\.\\d+') 20 newline_re = re.compile('(\\r\\n|\\r|\\n)') 21 TOKEN_ADD = intern('add') 22 TOKEN_ASSIGN = intern('assign') 23 TOKEN_COLON = intern('colon') 24 TOKEN_COMMA = intern('comma') 25 TOKEN_DIV = intern('div') 26 TOKEN_DOT = intern('dot') 27 TOKEN_EQ = intern('eq') 28 TOKEN_FLOORDIV = intern('floordiv') 29 TOKEN_GT = intern('gt') 30 TOKEN_GTEQ = intern('gteq') 31 TOKEN_LBRACE = intern('lbrace') 32 TOKEN_LBRACKET = intern('lbracket') 33 TOKEN_LPAREN = intern('lparen') 34 TOKEN_LT = intern('lt') 35 TOKEN_LTEQ = intern('lteq') 36 TOKEN_MOD = intern('mod') 37 TOKEN_MUL = intern('mul') 38 TOKEN_NE = intern('ne') 39 TOKEN_PIPE = intern('pipe') 40 TOKEN_POW = intern('pow') 41 TOKEN_RBRACE = intern('rbrace') 42 TOKEN_RBRACKET = intern('rbracket') 43 TOKEN_RPAREN = intern('rparen') 44 TOKEN_SEMICOLON = intern('semicolon') 45 TOKEN_SUB = intern('sub') 46 TOKEN_TILDE = intern('tilde') 47 TOKEN_WHITESPACE = intern('whitespace') 48 TOKEN_FLOAT = intern('float') 49 TOKEN_INTEGER = intern('integer') 50 TOKEN_NAME = intern('name') 51 TOKEN_STRING = intern('string') 52 TOKEN_OPERATOR = intern('operator') 53 TOKEN_BLOCK_BEGIN = intern('block_begin') 54 TOKEN_BLOCK_END = intern('block_end') 55 TOKEN_VARIABLE_BEGIN = intern('variable_begin') 56 TOKEN_VARIABLE_END = intern('variable_end') 57 TOKEN_RAW_BEGIN = intern('raw_begin') 58 TOKEN_RAW_END = intern('raw_end') 59 TOKEN_COMMENT_BEGIN = intern('comment_begin') 60 TOKEN_COMMENT_END = intern('comment_end') 61 TOKEN_COMMENT = intern('comment') 62 TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin') 63 TOKEN_LINESTATEMENT_END = intern('linestatement_end') 64 TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin') 65 TOKEN_LINECOMMENT_END = intern('linecomment_end') 66 TOKEN_LINECOMMENT = intern('linecomment') 67 TOKEN_DATA = intern('data') 68 TOKEN_INITIAL = intern('initial') 69 TOKEN_EOF = intern('eof') 70 operators = {'+': TOKEN_ADD, 71 '-': TOKEN_SUB, 72 '/': TOKEN_DIV, 73 '//': TOKEN_FLOORDIV, 74 '*': TOKEN_MUL, 75 '%': TOKEN_MOD, 76 '**': TOKEN_POW, 77 '~': TOKEN_TILDE, 78 '[': TOKEN_LBRACKET, 79 ']': TOKEN_RBRACKET, 80 '(': TOKEN_LPAREN, 81 ')': TOKEN_RPAREN, 82 '{': TOKEN_LBRACE, 83 '}': TOKEN_RBRACE, 84 '==': TOKEN_EQ, 85 '!=': TOKEN_NE, 86 '>': TOKEN_GT, 87 '>=': TOKEN_GTEQ, 88 '<': TOKEN_LT, 89 '<=': TOKEN_LTEQ, 90 '=': TOKEN_ASSIGN, 91 '.': TOKEN_DOT, 92 ':': TOKEN_COLON, 93 '|': TOKEN_PIPE, 94 ',': TOKEN_COMMA, 95 ';': TOKEN_SEMICOLON} 96 reverse_operators = dict([ (v, k) for k, v in operators.iteritems() ]) 97 operator_re = re.compile('(%s)' % '|'.join((re.escape(x) for x in sorted(operators, key=lambda x: -len(x))))) 98 ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, 99 TOKEN_COMMENT, 100 TOKEN_COMMENT_END, 101 TOKEN_WHITESPACE, 102 TOKEN_WHITESPACE, 103 TOKEN_LINECOMMENT_BEGIN, 104 TOKEN_LINECOMMENT_END, 105 TOKEN_LINECOMMENT]) 106 ignore_if_empty = frozenset([TOKEN_WHITESPACE, 107 TOKEN_DATA, 108 TOKEN_COMMENT, 109 TOKEN_LINECOMMENT]) 110 111 def _describe_token_type(token_type): 112 if token_type in reverse_operators: 113 return reverse_operators[token_type] 114 return {TOKEN_COMMENT_BEGIN: 'begin of comment', 115 TOKEN_COMMENT_END: 'end of comment', 116 TOKEN_COMMENT: 'comment', 117 TOKEN_LINECOMMENT: 'comment', 118 TOKEN_BLOCK_BEGIN: 'begin of statement block', 119 TOKEN_BLOCK_END: 'end of statement block', 120 TOKEN_VARIABLE_BEGIN: 'begin of print statement', 121 TOKEN_VARIABLE_END: 'end of print statement', 122 TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement', 123 TOKEN_LINESTATEMENT_END: 'end of line statement', 124 TOKEN_DATA: 'template data / text', 125 TOKEN_EOF: 'end of template'}.get(token_type, token_type) 126 127 128 def describe_token(token): 129 if token.type == 'name': 130 return token.value 131 return _describe_token_type(token.type) 132 133 134 def describe_token_expr(expr): 135 if ':' in expr: 136 type, value = expr.split(':', 1) 137 if type == 'name': 138 return value 139 else: 140 type = expr 141 return _describe_token_type(type) 142 143 144 def count_newlines(value): 145 return len(newline_re.findall(value)) 146 147 148 def compile_rules(environment): 149 e = re.escape 150 rules = [(len(environment.comment_start_string), 'comment', e(environment.comment_start_string)), (len(environment.block_start_string), 'block', e(environment.block_start_string)), (len(environment.variable_start_string), 'variable', e(environment.variable_start_string))] 151 if environment.line_statement_prefix is not None: 152 rules.append((len(environment.line_statement_prefix), 'linestatement', '^\\s*' + e(environment.line_statement_prefix))) 153 if environment.line_comment_prefix is not None: 154 rules.append((len(environment.line_comment_prefix), 'linecomment', '(?:^|(?<=\\S))[^\\S\\r\\n]*' + e(environment.line_comment_prefix))) 155 return [ x[1:] for x in sorted(rules, reverse=True) ] 156 157 158 class Failure(object): 159 160 def __init__(self, message, cls = TemplateSyntaxError): 161 self.message = message 162 self.error_class = cls 163 164 def __call__(self, lineno, filename): 165 raise self.error_class(self.message, lineno, filename) 166 167 168 class Token(tuple): 169 __slots__ = () 170 lineno, type, value = (property(itemgetter(x)) for x in range(3)) 171 172 def __new__(cls, lineno, type, value): 173 return tuple.__new__(cls, (lineno, intern(str(type)), value)) 174 175 def __str__(self): 176 if self.type in reverse_operators: 177 return reverse_operators[self.type] 178 if self.type == 'name': 179 return self.value 180 return self.type 181 182 def test(self, expr): 183 if self.type == expr: 184 return True 185 if ':' in expr: 186 return expr.split(':', 1) == [self.type, self.value] 187 return False 188 189 def test_any(self, *iterable): 190 for expr in iterable: 191 if self.test(expr): 192 return True 193 194 return False 195 196 def __repr__(self): 197 return 'Token(%r, %r, %r)' % (self.lineno, self.type, self.value) 198 199 200 class TokenStreamIterator(object): 201 202 def __init__(self, stream): 203 self.stream = stream 204 205 def __iter__(self): 206 return self 207 208 def next(self): 209 token = self.stream.current 210 if token.type is TOKEN_EOF: 211 self.stream.close() 212 raise StopIteration() 213 next(self.stream) 214 return token 215 216 217 class TokenStream(object): 218 219 def __init__(self, generator, name, filename): 220 self._next = iter(generator).next 221 self._pushed = deque() 222 self.name = name 223 self.filename = filename 224 self.closed = False 225 self.current = Token(1, TOKEN_INITIAL, '') 226 next(self) 227 228 def __iter__(self): 229 return TokenStreamIterator(self) 230 231 def __nonzero__(self): 232 return bool(self._pushed) or self.current.type is not TOKEN_EOF 233 234 eos = property(lambda x: not x, doc='Are we at the end of the stream?') 235 236 def push(self, token): 237 self._pushed.append(token) 238 239 def look(self): 240 old_token = next(self) 241 result = self.current 242 self.push(result) 243 self.current = old_token 244 return result 245 246 def skip(self, n = 1): 247 for x in xrange(n): 248 next(self) 249 250 def next_if(self, expr): 251 if self.current.test(expr): 252 return next(self) 253 254 def skip_if(self, expr): 255 return self.next_if(expr) is not None 256 257 def next(self): 258 rv = self.current 259 if self._pushed: 260 self.current = self._pushed.popleft() 261 elif self.current.type is not TOKEN_EOF: 262 try: 263 self.current = self._next() 264 except StopIteration: 265 self.close() 266 267 return rv 268 269 def close(self): 270 self.current = Token(self.current.lineno, TOKEN_EOF, '') 271 self._next = None 272 self.closed = True 273 274 def expect(self, expr): 275 if not self.current.test(expr): 276 expr = describe_token_expr(expr) 277 if self.current.type is TOKEN_EOF: 278 raise TemplateSyntaxError('unexpected end of template, expected %r.' % expr, self.current.lineno, self.name, self.filename) 279 raise TemplateSyntaxError('expected token %r, got %r' % (expr, describe_token(self.current)), self.current.lineno, self.name, self.filename) 280 try: 281 return self.current 282 finally: 283 next(self) 284 285 286 def get_lexer(environment): 287 key = (environment.block_start_string, 288 environment.block_end_string, 289 environment.variable_start_string, 290 environment.variable_end_string, 291 environment.comment_start_string, 292 environment.comment_end_string, 293 environment.line_statement_prefix, 294 environment.line_comment_prefix, 295 environment.trim_blocks, 296 environment.newline_sequence) 297 lexer = _lexer_cache.get(key) 298 if lexer is None: 299 lexer = Lexer(environment) 300 _lexer_cache[key] = lexer 301 return lexer 302 303 304 class Lexer(object): 305 306 def __init__(self, environment): 307 c = lambda x: re.compile(x, re.M | re.S) 308 e = re.escape 309 tag_rules = [(whitespace_re, TOKEN_WHITESPACE, None), 310 (float_re, TOKEN_FLOAT, None), 311 (integer_re, TOKEN_INTEGER, None), 312 (name_re, TOKEN_NAME, None), 313 (string_re, TOKEN_STRING, None), 314 (operator_re, TOKEN_OPERATOR, None)] 315 root_tag_rules = compile_rules(environment) 316 block_suffix_re = environment.trim_blocks and '\\n?' or '' 317 self.newline_sequence = environment.newline_sequence 318 self.rules = {'root': [(c('(.*?)(?:%s)' % '|'.join(['(?P<raw_begin>(?:\\s*%s\\-|%s)\\s*raw\\s*(?:\\-%s\\s*|%s))' % (e(environment.block_start_string), 319 e(environment.block_start_string), 320 e(environment.block_end_string), 321 e(environment.block_end_string))] + [ '(?P<%s_begin>\\s*%s\\-|%s)' % (n, r, r) for n, r in root_tag_rules ])), (TOKEN_DATA, '#bygroup'), '#bygroup'), (c('.+'), TOKEN_DATA, None)], 322 TOKEN_COMMENT_BEGIN: [(c('(.*?)((?:\\-%s\\s*|%s)%s)' % (e(environment.comment_end_string), e(environment.comment_end_string), block_suffix_re)), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'), (c('(.)'), (Failure('Missing end of comment tag'),), None)], 323 TOKEN_BLOCK_BEGIN: [(c('(?:\\-%s\\s*|%s)%s' % (e(environment.block_end_string), e(environment.block_end_string), block_suffix_re)), TOKEN_BLOCK_END, '#pop')] + tag_rules, 324 TOKEN_VARIABLE_BEGIN: [(c('\\-%s\\s*|%s' % (e(environment.variable_end_string), e(environment.variable_end_string))), TOKEN_VARIABLE_END, '#pop')] + tag_rules, 325 TOKEN_RAW_BEGIN: [(c('(.*?)((?:\\s*%s\\-|%s)\\s*endraw\\s*(?:\\-%s\\s*|%s%s))' % (e(environment.block_start_string), 326 e(environment.block_start_string), 327 e(environment.block_end_string), 328 e(environment.block_end_string), 329 block_suffix_re)), (TOKEN_DATA, TOKEN_RAW_END), '#pop'), (c('(.)'), (Failure('Missing end of raw directive'),), None)], 330 TOKEN_LINESTATEMENT_BEGIN: [(c('\\s*(\\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')] + tag_rules, 331 TOKEN_LINECOMMENT_BEGIN: [(c('(.*?)()(?=\\n|$)'), (TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END), '#pop')]} 332 333 def _normalize_newlines(self, value): 334 return newline_re.sub(self.newline_sequence, value) 335 336 def tokenize(self, source, name = None, filename = None, state = None): 337 stream = self.tokeniter(source, name, filename, state) 338 return TokenStream(self.wrap(stream, name, filename), name, filename) 339 340 def wrap(self, stream, name = None, filename = None): 341 for lineno, token, value in stream: 342 if token in ignored_tokens: 343 continue 344 elif token == 'linestatement_begin': 345 token = 'block_begin' 346 elif token == 'linestatement_end': 347 token = 'block_end' 348 elif token in ('raw_begin', 'raw_end'): 349 continue 350 elif token == 'data': 351 value = self._normalize_newlines(value) 352 elif token == 'keyword': 353 token = value 354 elif token == 'name': 355 value = str(value) 356 elif token == 'string': 357 try: 358 value = self._normalize_newlines(value[1:-1]).encode('ascii', 'backslashreplace').decode('unicode-escape') 359 except Exception as e: 360 msg = str(e).split(':')[-1].strip() 361 raise TemplateSyntaxError(msg, lineno, name, filename) 362 363 try: 364 value = str(value) 365 except UnicodeError: 366 pass 367 368 elif token == 'integer': 369 value = int(value) 370 elif token == 'float': 371 value = float(value) 372 elif token == 'operator': 373 token = operators[value] 374 yield Token(lineno, token, value) 375 376 def tokeniter(self, source, name, filename = None, state = None): 377 source = '\n'.join(unicode(source).splitlines()) 378 pos = 0 379 lineno = 1 380 stack = ['root'] 381 if state is not None and state != 'root': 382 stack.append(state + '_begin') 383 else: 384 state = 'root' 385 statetokens = self.rules[stack[-1]] 386 source_length = len(source) 387 balancing_stack = [] 388 while 1: 389 for regex, tokens, new_state in statetokens: 390 m = regex.match(source, pos) 391 if m is None: 392 continue 393 if balancing_stack and tokens in ('variable_end', 'block_end', 'linestatement_end'): 394 continue 395 if isinstance(tokens, tuple): 396 for idx, token in enumerate(tokens): 397 if token.__class__ is Failure: 398 raise token(lineno, filename) 399 elif token == '#bygroup': 400 for key, value in m.groupdict().iteritems(): 401 if value is not None: 402 yield (lineno, key, value) 403 lineno += value.count('\n') 404 break 405 else: 406 raise RuntimeError('%r wanted to resolve the token dynamically but no group matched' % regex) 407 408 else: 409 data = m.group(idx + 1) 410 if data or token not in ignore_if_empty: 411 yield (lineno, token, data) 412 lineno += data.count('\n') 413 414 else: 415 data = m.group() 416 if tokens == 'operator': 417 if data == '{': 418 balancing_stack.append('}') 419 elif data == '(': 420 balancing_stack.append(')') 421 elif data == '[': 422 balancing_stack.append(']') 423 elif data in ('}', ')', ']'): 424 if not balancing_stack: 425 raise TemplateSyntaxError("unexpected '%s'" % data, lineno, name, filename) 426 expected_op = balancing_stack.pop() 427 if expected_op != data: 428 raise TemplateSyntaxError("unexpected '%s', expected '%s'" % (data, expected_op), lineno, name, filename) 429 if data or tokens not in ignore_if_empty: 430 yield (lineno, tokens, data) 431 lineno += data.count('\n') 432 pos2 = m.end() 433 if new_state is not None: 434 if new_state == '#pop': 435 stack.pop() 436 elif new_state == '#bygroup': 437 for key, value in m.groupdict().iteritems(): 438 if value is not None: 439 stack.append(key) 440 break 441 else: 442 raise RuntimeError('%r wanted to resolve the new state dynamically but no group matched' % regex) 443 444 else: 445 stack.append(new_state) 446 statetokens = self.rules[stack[-1]] 447 elif pos2 == pos: 448 raise RuntimeError('%r yielded empty string without stack change' % regex) 449 pos = pos2 450 break 451 else: 452 if pos >= source_length: 453 return 454 raise TemplateSyntaxError('unexpected char %r at %d' % (source[pos], pos), lineno, name, filename)