parser.py (12858B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\localization\parser.py 2 from future.utils import raise_ 3 import re 4 import eveLocalization 5 from pytelemetry import zoning as telemetry 6 ARG_TO_MODIFIER = {'linkify': eveLocalization.TOKEN_FLAG.LINKIFY, 7 'capitalize': eveLocalization.TOKEN_FLAG.CAPITALIZE, 8 'uppercase': eveLocalization.TOKEN_FLAG.UPPERCASE, 9 'lowercase': eveLocalization.TOKEN_FLAG.LOWERCASE, 10 'titlecase': eveLocalization.TOKEN_FLAG.TITLECASE, 11 'useGrouping': eveLocalization.TOKEN_FLAG.USEGROUPING} 12 VARIABLE_TYPE_CHOOSER = {'generic': eveLocalization.VARIABLE_TYPE.GENERIC, 13 'character': eveLocalization.VARIABLE_TYPE.CHARACTER, 14 'npcorganization': eveLocalization.VARIABLE_TYPE.NPCORGANIZATION, 15 'item': eveLocalization.VARIABLE_TYPE.ITEM, 16 'location': eveLocalization.VARIABLE_TYPE.LOCATION, 17 'characterlist': eveLocalization.VARIABLE_TYPE.CHARACTERLIST, 18 'messageid': eveLocalization.VARIABLE_TYPE.MESSAGE, 19 'datetime': eveLocalization.VARIABLE_TYPE.DATETIME, 20 'formattedtime': eveLocalization.VARIABLE_TYPE.FORMATTEDTIME, 21 'timeinterval': eveLocalization.VARIABLE_TYPE.TIMEINTERVAL, 22 'numeric': eveLocalization.VARIABLE_TYPE.NUMERIC} 23 topLevelTagsRE = re.compile('(\\{(?!\\{).*?\\})') 24 tokensRE = re.compile('(""|"[^"]+"|->|[:.,=\\[\\]])') 25 validVariableNameRE = re.compile('[a-zA-Z_]+\\w?') 26 validAttributeNameRE = re.compile('[\\.[a-zA-Z_]+\\w*]*') 27 quoteWrappedStringRE = re.compile('"[^"]*"') 28 29 @telemetry.ZONE_FUNCTION 30 def _Tokenize(sourceText): 31 sourceText = sourceText.replace('\r\n', ' ').replace('\n', ' ') 32 tags = dict([ (each, None) for each in topLevelTagsRE.findall(sourceText) ]) 33 for tag in tags.keys()[:]: 34 errors = [] 35 originalTag = tag 36 tag = tag[1:-1] 37 if not len(tag.strip()): 38 del tags[originalTag] 39 continue 40 tokens = tokensRE.split(tag) 41 originalTokens = [ token for token in tokens if token ] 42 tokens = [ token.strip() for token in tokens if token and token.strip() ] 43 dp = None 44 tp = None 45 try: 46 variableName = None 47 variableType = 'generic' 48 propertyName = None 49 tagArgs = 0 50 tagKwargs = {} 51 conditionalValues = [] 52 token = tokens.pop(0) 53 if token == '[': 54 variableType = tokens.pop(0) 55 if variableType not in VARIABLE_TYPE_DATA: 56 raise_(SyntaxError, "'" + variableType + "' is not a valid variable type") 57 next = tokens.pop(0) 58 if next != ']': 59 raise_(SyntaxError, "Expected ']' but got '%s'" % next) 60 variableName = tokens.pop(0) 61 else: 62 variableType = 'generic' 63 variableName = token 64 if not _IsValidVariableName(variableName): 65 raise SyntaxError('Variable names must be alphanumeric, and begin with a letter or underscore') 66 for tagName, tagInfo in [ (tagName, tagInfo) for tagName, tagInfo in tags.iteritems() if tagInfo ]: 67 if tagInfo['variableName'] == variableName and tagInfo['variableType'] != VARIABLE_TYPE_CHOOSER[variableType.lower().encode('ascii')]: 68 raise_(TypeError, "'" + variableName + "' already declared as type '" + tagInfo['variableType'] + "'.") 69 70 if tokens: 71 next = tokens.pop(0) 72 if next == '.': 73 propertyName = tokens.pop(0) 74 if variableType == 'numeric': 75 tagArgs |= eveLocalization.TOKEN_FLAG.QUANTITY 76 if tokens: 77 next = tokens.pop(0) 78 if tokens: 79 if next == '->': 80 tagArgs |= eveLocalization.TOKEN_FLAG.CONDITIONAL 81 if VARIABLE_TYPE_DATA[variableType.encode('ascii')]['conditionalProperties'][propertyName] == CONDITIONAL_TYPE_QUANTITY: 82 tagArgs |= eveLocalization.TOKEN_FLAG.QUANTITY 83 if variableType in VARIABLE_TYPE_DATA and 'conditionalProperties' in VARIABLE_TYPE_DATA[variableType] and propertyName not in VARIABLE_TYPE_DATA[variableType]['conditionalProperties']: 84 raise_(TypeError, 'You cannot use ' + unicode(variableName) + '.' + unicode(propertyName) + ' in conditional statements. Valid properties for this variable type are: ' + str(VARIABLE_TYPE_DATA[variableType]['conditionalProperties'])) 85 conditionalValue = tokens.pop(0) 86 if not _IsQuoteWrappedString(conditionalValue): 87 raise_(SyntaxError, "Error parsing conditional value '" + conditionalValue + "'") 88 conditionalValues.append(conditionalValue[1:-1]) 89 while tokens: 90 next = tokens.pop(0) 91 if next != ',': 92 raise_(SyntaxError, "Expected ',' but got '%s'" % next) 93 conditionalValue = tokens.pop(0) 94 if not _IsQuoteWrappedString(conditionalValue): 95 raise_(SyntaxError, "Error parsing conditional value '" + conditionalValue + "'") 96 conditionalValues.append(conditionalValue[1:-1]) 97 98 elif next == ',': 99 while tokens: 100 arg = tokens.pop(0).encode('ascii') 101 next = tokens.pop(0) if len(tokens) else None 102 if next in (',', None): 103 if arg not in VARIABLE_TYPE_DATA[variableType]['args']: 104 raise_(SyntaxError, "Modifier '" + arg + "' cannot be used with variables of type '" + variableType + "'") 105 tagArgs |= ARG_TO_MODIFIER[arg] 106 elif next == '=': 107 kwargValue = tokens.pop(0) 108 if _IsQuoteWrappedString(kwargValue): 109 kwargValue = kwargValue[1:-1] 110 elif not kwargValue.isalnum(): 111 raise ValueError('Conditional arguments must be an alphanumeric value or double-quoted string') 112 if arg not in VARIABLE_TYPE_DATA[variableType]['kwargs']: 113 raise_(SyntaxError, "Modifier '" + arg + "' cannot be used with variables of type '" + variableType + "'") 114 if arg in ('decimalPlaces', 'leadingZeroes'): 115 tagKwargs[arg] = int(kwargValue) 116 if arg == 'decimalPlaces': 117 tagArgs |= eveLocalization.TOKEN_FLAG.DECIMALPLACES 118 elif arg == 'timeFormat': 119 if kwargValue == 'short': 120 tagKwargs['format'] = u'%Y.%m.%d %H:%M' 121 else: 122 tagKwargs['format'] = u'%Y.%m.%d %H:%M:%S' 123 elif arg == 'format': 124 tagKwargs[arg] = unicode(kwargValue) 125 elif arg in ('date', 'time') and variableType.lower() == 'datetime': 126 if arg == 'date': 127 if kwargValue == 'long': 128 dp = u'%B %d, %Y' 129 elif kwargValue == 'full': 130 dp = u'%#c' 131 elif kwargValue == 'medium': 132 dp = u'%b %d, %Y' 133 elif kwargValue == 'short': 134 dp = u'%Y.%m.%d' 135 elif kwargValue == 'short': 136 tp = u'%H:%M' 137 elif kwargValue.lower() != 'none': 138 tp = u'%H:%M:%S' 139 if dp and tp: 140 if dp == u'%#c': 141 if tp != '%H:%M': 142 tagKwargs['format'] = dp 143 else: 144 tagKwargs['format'] = u'%#x ' + tp 145 else: 146 tagKwargs['format'] = dp + u' ' + tp 147 elif dp and not tp: 148 if dp == u'%#c': 149 tagKwargs['format'] = u'%#x' 150 else: 151 tagKwargs['format'] = dp 152 elif tp and not dp: 153 tagKwargs['format'] = tp 154 else: 155 tagKwargs[arg] = kwargValue.encode('ascii') 156 next = tokens.pop(0) if len(tokens) else None 157 if next not in (',', None): 158 raise_(SyntaxError, "Expected ',' but got '%s'" % next) 159 else: 160 raise_(SyntaxError, "Expected ',' or '=' but got '%s'" % next) 161 162 else: 163 raise_(SyntaxError, "Expected ',' but got '%s'" % next) 164 except (TypeError, 165 ValueError, 166 SyntaxError, 167 IndexError) as e: 168 partialTag = '{' + ''.join([ originalTokens[i] for i in xrange(len(originalTokens) - len(tokens)) ]) 169 message1 = 'Tokenizer error: ' + repr(e) + ' -- Stopped parsing at: ' + partialTag 170 message2 = "Source tag '" + originalTag + "' Source string: '" + sourceText + "'" 171 errors.append(message1) 172 errors.append(message2) 173 raise 174 175 tags[originalTag] = {'variableType': VARIABLE_TYPE_CHOOSER[variableType.lower().encode('ascii')], 176 'variableName': variableName.encode('ascii') if variableName is not None else None, 177 'propertyName': propertyName.encode('ascii') if propertyName is not None else None, 178 'conditionalValues': conditionalValues, 179 'args': tagArgs, 180 'kwargs': tagKwargs} 181 182 return tags 183 184 185 def _IsValidVariableName(string): 186 results = validVariableNameRE.findall(string) 187 return results and results[0] == string 188 189 190 def _IsValidAttributeName(string): 191 results = validVariableNameRE.findall(string) 192 return results and results[0] == string 193 194 195 def _IsQuoteWrappedString(string): 196 matches = quoteWrappedStringRE.findall(string) 197 return len(matches) and matches[0] == string 198 199 200 def _FormatErrorMessage(args): 201 return ''.join(map(unicode, args)) 202 203 204 BASIC_TEXT_MODIFIERS = ['capitalize', 205 'uppercase', 206 'lowercase', 207 'titlecase'] 208 BASIC_TEXT_SETTINGS = ['linkinfo'] 209 CONDITIONAL_TYPE_GENDER = 'gender' 210 CONDITIONAL_TYPE_GENDERS = 'genders' 211 CONDITIONAL_TYPE_QUANTITY = 'quantity' 212 VARIABLE_TYPE_DATA = {'character': {'args': BASIC_TEXT_MODIFIERS + ['linkify'], 213 'kwargs': BASIC_TEXT_SETTINGS, 214 'conditionalProperties': {'gender': CONDITIONAL_TYPE_GENDER}}, 215 'npcOrganization': {'args': BASIC_TEXT_MODIFIERS + ['linkify'], 216 'kwargs': BASIC_TEXT_SETTINGS, 217 'conditionalProperties': {'gender': CONDITIONAL_TYPE_GENDER}}, 218 'item': {'args': BASIC_TEXT_MODIFIERS + ['linkify'], 219 'kwargs': BASIC_TEXT_SETTINGS + ['quantity'], 220 'conditionalProperties': {'gender': CONDITIONAL_TYPE_GENDER, 221 'quantity': CONDITIONAL_TYPE_QUANTITY}}, 222 'location': {'args': BASIC_TEXT_MODIFIERS + ['linkify'], 223 'kwargs': BASIC_TEXT_SETTINGS, 224 'conditionalProperties': {'gender': CONDITIONAL_TYPE_GENDER}}, 225 'characterlist': {'args': [], 226 'kwargs': BASIC_TEXT_SETTINGS, 227 'conditionalProperties': {'genders': CONDITIONAL_TYPE_GENDERS, 228 'quantity': CONDITIONAL_TYPE_QUANTITY}}, 229 'messageid': {'args': BASIC_TEXT_MODIFIERS, 230 'kwargs': BASIC_TEXT_SETTINGS}, 231 'datetime': {'args': BASIC_TEXT_MODIFIERS, 232 'kwargs': BASIC_TEXT_SETTINGS + ['date', 'time']}, 233 'formattedtime': {'args': BASIC_TEXT_MODIFIERS, 234 'kwargs': BASIC_TEXT_SETTINGS + ['format']}, 235 'timeinterval': {'args': BASIC_TEXT_MODIFIERS, 236 'kwargs': BASIC_TEXT_SETTINGS + ['from', 'to']}, 237 'numeric': {'args': ['useGrouping'], 238 'kwargs': BASIC_TEXT_SETTINGS + ['decimalPlaces', 'leadingZeroes'], 239 'conditionalProperties': {None: CONDITIONAL_TYPE_QUANTITY}}, 240 'generic': {'args': BASIC_TEXT_MODIFIERS, 241 'kwargs': BASIC_TEXT_SETTINGS}}