ext.py (14748B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\lib\jinja2\ext.py 2 from collections import deque 3 from jinja2 import nodes 4 from jinja2.defaults import * 5 from jinja2.environment import Environment 6 from jinja2.runtime import Undefined, concat 7 from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError 8 from jinja2.utils import contextfunction, import_string, Markup, next 9 GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext') 10 11 class ExtensionRegistry(type): 12 13 def __new__(cls, name, bases, d): 14 rv = type.__new__(cls, name, bases, d) 15 rv.identifier = rv.__module__ + '.' + rv.__name__ 16 return rv 17 18 19 class Extension(object): 20 __metaclass__ = ExtensionRegistry 21 tags = set() 22 priority = 100 23 24 def __init__(self, environment): 25 self.environment = environment 26 27 def bind(self, environment): 28 rv = object.__new__(self.__class__) 29 rv.__dict__.update(self.__dict__) 30 rv.environment = environment 31 return rv 32 33 def preprocess(self, source, name, filename = None): 34 return source 35 36 def filter_stream(self, stream): 37 return stream 38 39 def parse(self, parser): 40 raise NotImplementedError() 41 42 def attr(self, name, lineno = None): 43 return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno) 44 45 def call_method(self, name, args = None, kwargs = None, dyn_args = None, dyn_kwargs = None, lineno = None): 46 if args is None: 47 args = [] 48 if kwargs is None: 49 kwargs = [] 50 return nodes.Call(self.attr(name, lineno=lineno), args, kwargs, dyn_args, dyn_kwargs, lineno=lineno) 51 52 53 @contextfunction 54 def _gettext_alias(__context, *args, **kwargs): 55 return __context.call(__context.resolve('gettext'), *args, **kwargs) 56 57 58 def _make_new_gettext(func): 59 60 @contextfunction 61 def gettext(__context, __string, **variables): 62 rv = __context.call(func, __string) 63 if __context.eval_ctx.autoescape: 64 rv = Markup(rv) 65 return rv % variables 66 67 return gettext 68 69 70 def _make_new_ngettext(func): 71 72 @contextfunction 73 def ngettext(__context, __singular, __plural, __num, **variables): 74 variables.setdefault('num', __num) 75 rv = __context.call(func, __singular, __plural, __num) 76 if __context.eval_ctx.autoescape: 77 rv = Markup(rv) 78 return rv % variables 79 80 return ngettext 81 82 83 class InternationalizationExtension(Extension): 84 tags = set(['trans']) 85 86 def __init__(self, environment): 87 Extension.__init__(self, environment) 88 environment.globals['_'] = _gettext_alias 89 environment.extend(install_gettext_translations=self._install, install_null_translations=self._install_null, install_gettext_callables=self._install_callables, uninstall_gettext_translations=self._uninstall, extract_translations=self._extract, newstyle_gettext=False) 90 91 def _install(self, translations, newstyle = None): 92 gettext = getattr(translations, 'ugettext', None) 93 if gettext is None: 94 gettext = translations.gettext 95 ngettext = getattr(translations, 'ungettext', None) 96 if ngettext is None: 97 ngettext = translations.ngettext 98 self._install_callables(gettext, ngettext, newstyle) 99 100 def _install_null(self, newstyle = None): 101 self._install_callables(lambda x: x, lambda s, p, n: (n != 1 and (p,) or (s,))[0], newstyle) 102 103 def _install_callables(self, gettext, ngettext, newstyle = None): 104 if newstyle is not None: 105 self.environment.newstyle_gettext = newstyle 106 if self.environment.newstyle_gettext: 107 gettext = _make_new_gettext(gettext) 108 ngettext = _make_new_ngettext(ngettext) 109 self.environment.globals.update(gettext=gettext, ngettext=ngettext) 110 111 def _uninstall(self, translations): 112 for key in ('gettext', 'ngettext'): 113 self.environment.globals.pop(key, None) 114 115 def _extract(self, source, gettext_functions = GETTEXT_FUNCTIONS): 116 if isinstance(source, basestring): 117 source = self.environment.parse(source) 118 return extract_from_ast(source, gettext_functions) 119 120 def parse(self, parser): 121 lineno = next(parser.stream).lineno 122 num_called_num = False 123 plural_expr = None 124 variables = {} 125 while parser.stream.current.type != 'block_end': 126 if variables: 127 parser.stream.expect('comma') 128 if parser.stream.skip_if('colon'): 129 break 130 name = parser.stream.expect('name') 131 if name.value in variables: 132 parser.fail('translatable variable %r defined twice.' % name.value, name.lineno, exc=TemplateAssertionError) 133 if parser.stream.current.type == 'assign': 134 next(parser.stream) 135 variables[name.value] = var = parser.parse_expression() 136 else: 137 variables[name.value] = var = nodes.Name(name.value, 'load') 138 if plural_expr is None: 139 plural_expr = var 140 num_called_num = name.value == 'num' 141 142 parser.stream.expect('block_end') 143 plural = plural_names = None 144 have_plural = False 145 referenced = set() 146 singular_names, singular = self._parse_block(parser, True) 147 if singular_names: 148 referenced.update(singular_names) 149 if plural_expr is None: 150 plural_expr = nodes.Name(singular_names[0], 'load') 151 num_called_num = singular_names[0] == 'num' 152 if parser.stream.current.test('name:pluralize'): 153 have_plural = True 154 next(parser.stream) 155 if parser.stream.current.type != 'block_end': 156 name = parser.stream.expect('name') 157 if name.value not in variables: 158 parser.fail('unknown variable %r for pluralization' % name.value, name.lineno, exc=TemplateAssertionError) 159 plural_expr = variables[name.value] 160 num_called_num = name.value == 'num' 161 parser.stream.expect('block_end') 162 plural_names, plural = self._parse_block(parser, False) 163 next(parser.stream) 164 referenced.update(plural_names) 165 else: 166 next(parser.stream) 167 for var in referenced: 168 if var not in variables: 169 variables[var] = nodes.Name(var, 'load') 170 171 if not have_plural: 172 plural_expr = None 173 elif plural_expr is None: 174 parser.fail('pluralize without variables', lineno) 175 node = self._make_node(singular, plural, variables, plural_expr, bool(referenced), num_called_num and have_plural) 176 node.set_lineno(lineno) 177 return node 178 179 def _parse_block(self, parser, allow_pluralize): 180 referenced = [] 181 buf = [] 182 while 1: 183 if parser.stream.current.type == 'data': 184 buf.append(parser.stream.current.value.replace('%', '%%')) 185 next(parser.stream) 186 elif parser.stream.current.type == 'variable_begin': 187 next(parser.stream) 188 name = parser.stream.expect('name').value 189 referenced.append(name) 190 buf.append('%%(%s)s' % name) 191 parser.stream.expect('variable_end') 192 elif parser.stream.current.type == 'block_begin': 193 next(parser.stream) 194 if parser.stream.current.test('name:endtrans'): 195 break 196 elif parser.stream.current.test('name:pluralize'): 197 if allow_pluralize: 198 break 199 parser.fail('a translatable section can have only one pluralize section') 200 parser.fail('control structures in translatable sections are not allowed') 201 elif parser.stream.eos: 202 parser.fail('unclosed translation block') 203 204 return (referenced, concat(buf)) 205 206 def _make_node(self, singular, plural, variables, plural_expr, vars_referenced, num_called_num): 207 if not vars_referenced and not self.environment.newstyle_gettext: 208 singular = singular.replace('%%', '%') 209 if plural: 210 plural = plural.replace('%%', '%') 211 if plural_expr is None: 212 gettext = nodes.Name('gettext', 'load') 213 node = nodes.Call(gettext, [nodes.Const(singular)], [], None, None) 214 else: 215 ngettext = nodes.Name('ngettext', 'load') 216 node = nodes.Call(ngettext, [nodes.Const(singular), nodes.Const(plural), plural_expr], [], None, None) 217 if self.environment.newstyle_gettext: 218 for key, value in variables.iteritems(): 219 if num_called_num and key == 'num': 220 continue 221 node.kwargs.append(nodes.Keyword(key, value)) 222 223 else: 224 node = nodes.MarkSafeIfAutoescape(node) 225 if variables: 226 node = nodes.Mod(node, nodes.Dict([ nodes.Pair(nodes.Const(key), value) for key, value in variables.items() ])) 227 return nodes.Output([node]) 228 229 230 class ExprStmtExtension(Extension): 231 tags = set(['do']) 232 233 def parse(self, parser): 234 node = nodes.ExprStmt(lineno=next(parser.stream).lineno) 235 node.node = parser.parse_tuple() 236 return node 237 238 239 class LoopControlExtension(Extension): 240 tags = set(['break', 'continue']) 241 242 def parse(self, parser): 243 token = next(parser.stream) 244 if token.value == 'break': 245 return nodes.Break(lineno=token.lineno) 246 return nodes.Continue(lineno=token.lineno) 247 248 249 class WithExtension(Extension): 250 tags = set(['with']) 251 252 def parse(self, parser): 253 node = nodes.Scope(lineno=next(parser.stream).lineno) 254 assignments = [] 255 while parser.stream.current.type != 'block_end': 256 lineno = parser.stream.current.lineno 257 if assignments: 258 parser.stream.expect('comma') 259 target = parser.parse_assign_target() 260 parser.stream.expect('assign') 261 expr = parser.parse_expression() 262 assignments.append(nodes.Assign(target, expr, lineno=lineno)) 263 264 node.body = assignments + list(parser.parse_statements(('name:endwith',), drop_needle=True)) 265 return node 266 267 268 class AutoEscapeExtension(Extension): 269 tags = set(['autoescape']) 270 271 def parse(self, parser): 272 node = nodes.ScopedEvalContextModifier(lineno=next(parser.stream).lineno) 273 node.options = [nodes.Keyword('autoescape', parser.parse_expression())] 274 node.body = parser.parse_statements(('name:endautoescape',), drop_needle=True) 275 return nodes.Scope([node]) 276 277 278 def extract_from_ast(node, gettext_functions = GETTEXT_FUNCTIONS, babel_style = True): 279 for node in node.find_all(nodes.Call): 280 if not isinstance(node.node, nodes.Name) or node.node.name not in gettext_functions: 281 continue 282 strings = [] 283 for arg in node.args: 284 if isinstance(arg, nodes.Const) and isinstance(arg.value, basestring): 285 strings.append(arg.value) 286 else: 287 strings.append(None) 288 289 for arg in node.kwargs: 290 strings.append(None) 291 292 if node.dyn_args is not None: 293 strings.append(None) 294 if node.dyn_kwargs is not None: 295 strings.append(None) 296 if not babel_style: 297 strings = tuple((x for x in strings if x is not None)) 298 if not strings: 299 continue 300 elif len(strings) == 1: 301 strings = strings[0] 302 else: 303 strings = tuple(strings) 304 yield (node.lineno, node.node.name, strings) 305 306 307 class _CommentFinder(object): 308 309 def __init__(self, tokens, comment_tags): 310 self.tokens = tokens 311 self.comment_tags = comment_tags 312 self.offset = 0 313 self.last_lineno = 0 314 315 def find_backwards(self, offset): 316 try: 317 for _, token_type, token_value in reversed(self.tokens[self.offset:offset]): 318 if token_type in ('comment', 'linecomment'): 319 try: 320 prefix, comment = token_value.split(None, 1) 321 except ValueError: 322 continue 323 324 if prefix in self.comment_tags: 325 return [comment.rstrip()] 326 327 return [] 328 finally: 329 self.offset = offset 330 331 def find_comments(self, lineno): 332 if not self.comment_tags or self.last_lineno > lineno: 333 return [] 334 for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]): 335 if token_lineno > lineno: 336 return self.find_backwards(self.offset + idx) 337 338 return self.find_backwards(len(self.tokens)) 339 340 341 def babel_extract(fileobj, keywords, comment_tags, options): 342 extensions = set() 343 for extension in options.get('extensions', '').split(','): 344 extension = extension.strip() 345 if not extension: 346 continue 347 extensions.add(import_string(extension)) 348 349 if InternationalizationExtension not in extensions: 350 extensions.add(InternationalizationExtension) 351 352 def getbool(options, key, default = False): 353 return options.get(key, str(default)).lower() in ('1', 'on', 'yes', 'true') 354 355 silent = getbool(options, 'silent', True) 356 environment = Environment(options.get('block_start_string', BLOCK_START_STRING), options.get('block_end_string', BLOCK_END_STRING), options.get('variable_start_string', VARIABLE_START_STRING), options.get('variable_end_string', VARIABLE_END_STRING), options.get('comment_start_string', COMMENT_START_STRING), options.get('comment_end_string', COMMENT_END_STRING), options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX, options.get('line_comment_prefix') or LINE_COMMENT_PREFIX, getbool(options, 'trim_blocks', TRIM_BLOCKS), NEWLINE_SEQUENCE, frozenset(extensions), cache_size=0, auto_reload=False) 357 if getbool(options, 'newstyle_gettext'): 358 environment.newstyle_gettext = True 359 source = fileobj.read().decode(options.get('encoding', 'utf-8')) 360 try: 361 node = environment.parse(source) 362 tokens = list(environment.lex(environment.preprocess(source))) 363 except TemplateSyntaxError as e: 364 if not silent: 365 raise 366 return 367 368 finder = _CommentFinder(tokens, comment_tags) 369 for lineno, func, message in extract_from_ast(node, keywords): 370 yield (lineno, 371 func, 372 message, 373 finder.find_comments(lineno)) 374 375 376 i18n = InternationalizationExtension 377 do = ExprStmtExtension 378 loopcontrols = LoopControlExtension 379 with_ = WithExtension 380 autoescape = AutoEscapeExtension