localizationBase.py (24797B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\localization\localizationBase.py 2 from __future__ import print_function 3 from __future__ import absolute_import 4 import cPickle as pickle 5 import hashlib 6 import logging 7 import re 8 import sys 9 import blue 10 import eveLocalization 11 import langutils 12 import logmodule 13 from carbon.common.script.sys.serviceConst import ROLEMASK_ELEVATEDPLAYER 14 from carbon.common.script.util import commonutils 15 from eveprefs import boot, prefs 16 from localization.common.util import UNRESOLVED_LOC_MARKER 17 from pytelemetry import zoning as telemetry 18 from localization import const as locconst, internalUtil, logger, parser, uiutil, util 19 from localization.settings import bilingualSettings, qaSettings 20 log = logging.getLogger(__name__) 21 OFFSET_TEXT = 0 22 OFFSET_METADATA = 1 23 OFFSET_TOKENS = 2 24 FALLBACK_TRANSLATION_STRINGS = {'en': u'Translation not available', 25 'ru': u'\u041f\u0435\u0440\u0435\u0432\u043e\u0434 \u043e\u0442\u0441\u0443\u0442\u0441\u0442\u0432\u0443\u0435\u0442', 26 'zh': u'\u5c1a\u672a\u63d0\u4f9b\u7ffb\u8bd1', 27 'fr': u'Traduction non disponible', 28 'de': u'\xdcbersetzung nicht verf\xfcgbar', 29 'ja': u'\u7ffb\u8a33\u306f\u307e\u3060\u7528\u610f\u3055\u308c\u3066\u3044\u307e\u305b\u3093', 30 'ko': u'\ubc88\uc5ed\uc774 \uc544\uc9c1 \uc81c\uacf5\ub418\uc9c0 \uc54a\uc558\uc2b5\ub2c8\ub2e4', 31 'es': u'Traducci\xf3n no disponible'} 32 33 class Localization(object): 34 __guid__ = 'localization.Localization' 35 PICKLE_EXT = '.pickle' 36 FSD_CLIENT_PREFIX = 'res:/localizationFSD/localization_fsd_' 37 FIELD_LABELS = 'labels' 38 FIELD_LANGUAGES = 'languages' 39 FIELD_TYPES = 'types' 40 41 def __init__(self): 42 self._InitializeInternalVariables() 43 self._InitializeTextData() 44 self._importantNameSetting = None 45 self._languageTooltipSetting = None 46 self._qaTooltipOverride = None 47 self._highlightImportantSetting = None 48 self._MISSING_LABELS_MODE = 0 49 message = u'Cerberus localization module loaded on ' + (boot.role if boot else '(Tool Environment)') 50 log.info(message) 51 print(message) 52 53 def SetFallBackMode(self, mode): 54 self._MISSING_LABELS_MODE = mode 55 56 def GetFallBackMode(self): 57 return self._MISSING_LABELS_MODE 58 59 def LoadLanguageData(self): 60 from . import propertyHandlers 61 if boot.role == 'client': 62 internalUtil.ClearLanguageID() 63 picklePrefixes = [(self.FSD_CLIENT_PREFIX, locconst.DATATYPE_FSD)] 64 supportedLanguagesByPrefix = {} 65 for prefix, dataType in picklePrefixes: 66 supportedLanguagesByPrefix[prefix] = self._ReadLocalizationMainPickle(prefix + 'main' + self.PICKLE_EXT, dataType) 67 if not supportedLanguagesByPrefix[prefix]: 68 message = u'Cerberus localization module failed to load MAIN pickle on %s. See log server for details.' % boot.role 69 log.error(message) 70 print(message) 71 return 72 73 self._ValidateAndRepairPrefs() 74 for prefix, dataType in picklePrefixes: 75 if not self._ReadLocalizationLanguagePickles(prefix, supportedLanguagesByPrefix[prefix], dataType): 76 message = u'Cerberus localization module failed to load language pickles on %s. See log server for details.' % boot.role 77 log.error(message) 78 print(message) 79 return 80 81 def GetLanguageStartupArg(self): 82 argKey = '/language=' 83 for arg in blue.pyos.GetArg()[1:]: 84 if arg and arg.startswith(argKey): 85 log.info(u'language argument from launcher found: %s', arg) 86 return arg[len(argKey):] 87 88 def GetLanguages(self): 89 return list(self.languagesDefined) 90 91 @telemetry.ZONE_METHOD 92 def LoadPrimaryLanguage(self, prefix, dataType): 93 languageID = self._GetPrimaryLanguage() 94 self._primaryLanguageID = languageID 95 eveLocalization.SetPrimaryLanguage(languageID) 96 if self._primaryLanguageID in self.languagesDefined: 97 if not self._LoadLanguagePickle(prefix, self._primaryLanguageID, dataType): 98 return False 99 else: 100 log.error(u"Language '%r' is not enabled. Text data was not loaded.", languageID) 101 return False 102 return True 103 104 def GetImportantByMessageID(self, messageID, **kwargs): 105 if boot.region == 'optic': 106 if not (session and session.role & ROLEMASK_ELEVATEDPLAYER): 107 return self.GetByMessageID(messageID, **kwargs) 108 if boot.role == 'proxy': 109 return self.GetByMessageID(messageID, **kwargs) 110 playerLanguageID = kwargs.pop('languageID', None) or internalUtil.GetLanguageID() 111 if playerLanguageID != self._primaryLanguageID or self._QaTooltipOverride(): 112 if self.UsePrimaryLanguageText(): 113 textString = self.GetByMessageID(messageID, self._primaryLanguageID, **kwargs) 114 hintLang = playerLanguageID 115 else: 116 textString = self.GetByMessageID(messageID, playerLanguageID, **kwargs) 117 hintLang = self._primaryLanguageID 118 if self.HighlightImportant(): 119 textString = '%s%s' % (textString, locconst.HIGHLIGHT_IMPORTANT_MARKER) 120 if self.UseImportantTooltip() and not qaSettings.LocWrapSettingsActive(): 121 if self._QaTooltipOverride() and playerLanguageID == self._primaryLanguageID: 122 hintString = textString[:-1][::-1] 123 else: 124 hintString = self.GetByMessageID(messageID, hintLang, **kwargs) 125 hintString = hintString.replace('"', "'").replace('<', '[').replace('>', ']') 126 textString = '<localized hint="%s">%s</localized>' % (hintString or '', textString) 127 else: 128 textString = self.GetByMessageID(messageID, **kwargs) 129 return textString 130 131 def GetImportantByLabel(self, labelNameAndPath, **kwargs): 132 try: 133 messageID = self.languageLabels[labelNameAndPath] 134 except KeyError: 135 logger.LogTraceback(u'No label with name %r', labelNameAndPath) 136 return '[no label: %s]' % labelNameAndPath 137 138 return self.GetImportantByMessageID(messageID, **kwargs) 139 140 def FormatImportantString(self, locText, englishText): 141 if boot.region == 'optic': 142 if not (session and session.role & ROLEMASK_ELEVATEDPLAYER): 143 return locText 144 playerLanguageID = internalUtil.GetLanguageID() 145 if playerLanguageID == self._primaryLanguageID and self._QaTooltipOverride(): 146 return '<localized hint="%s">%s*</localized>' % (englishText[::-1], englishText) 147 if playerLanguageID != self._primaryLanguageID: 148 if self.UsePrimaryLanguageText(): 149 textString = englishText 150 hintText = locText 151 else: 152 textString = locText 153 hintText = englishText 154 if self.HighlightImportant(): 155 textString = '%s*' % textString 156 if self.UseImportantTooltip() and not qaSettings.LocWrapSettingsActive(): 157 textString = '<localized hint="%s">%s</localized>' % (hintText, textString) 158 else: 159 textString = locText 160 return textString 161 162 def CleanImportantMarkup(self, textString): 163 if self.UseImportantTooltip(): 164 textString = commonutils.StripTags(textString, stripOnly=['localized']) 165 if self.HighlightImportant() and len(textString) > 1 and textString[-1] == '*': 166 textString = textString.replace(locconst.HIGHLIGHT_IMPORTANT_MARKER, '') 167 return textString 168 169 def Get(self, messageIDorLabel, languageID = None, **kwargs): 170 if not messageIDorLabel: 171 return 172 comfylang = langutils.any_to_comfy_language(languageID or internalUtil.GetLanguageID()) 173 if session and 'player' not in kwargs: 174 kwargs['player'] = session.charid 175 if isinstance(messageIDorLabel, (str, unicode)): 176 messageID = self.languageLabels.get(messageIDorLabel, None) 177 if not messageID: 178 if self._MISSING_LABELS_MODE: 179 return FALLBACK_TRANSLATION_STRINGS.get(comfylang.cerberus_code(), None) 180 else: 181 logger.LogTraceback(u'No label with name %r' % messageIDorLabel) 182 return 183 else: 184 messageID = messageIDorLabel 185 try: 186 textString = self._GetByMessageID(messageID, comfylang.cerberus_code(), **kwargs) 187 if '<localized' in textString: 188 textString = re.sub('^<localized hint="[a-z\\sA-Z]*">', '', textString) 189 textString = re.sub('<\\/localized>', '', textString) 190 return uiutil.PrepareLocalizationSafeString(textString, messageID=messageID) 191 except KeyError: 192 if self._MISSING_LABELS_MODE: 193 return FALLBACK_TRANSLATION_STRINGS.get(comfylang.cerberus_code(), None) 194 else: 195 return 196 except Exception as ex: 197 log.warning('_GetByMessageID failed: %r' % ex, exc_info=1, extra={k:v for k, v in sys._getframe(1).f_locals.iteritems() if isinstance(v, basestring) and isinstance(k, basestring) and 'label' in k.lower()}) 198 try: 199 return eveLocalization.GetRawByMessageID(messageID, comfylang.cerberus_code()) 200 except: 201 return 202 203 def GetByMessageID(self, messageID, languageID = None, **kwargs): 204 if messageID is None: 205 return '' 206 val = self.Get(messageID, languageID, **kwargs) 207 if val is None: 208 return u'%s %s]' % (UNRESOLVED_LOC_MARKER, messageID) 209 return val 210 211 def GetByLabel(self, labelNameAndPath, languageID = None, **kwargs): 212 val = self.Get(labelNameAndPath, languageID, **kwargs) 213 if val is None: 214 return u'[no label: %s]' % labelNameAndPath 215 return val 216 217 def GetMessageIDForLabel(self, labelNameAndPath): 218 messageID = self.languageLabels[labelNameAndPath] 219 return messageID 220 221 def IsValidMessageID(self, messageID, languageID = None): 222 languageID = util.StandardizedLanguageIDOrDefault(languageID) 223 return eveLocalization.IsValidMessageID(messageID, languageID) 224 225 def IsValidLabel(self, labelNameAndPath, languageID = None): 226 try: 227 messageID = self.languageLabels[labelNameAndPath] 228 return self.IsValidMessageID(messageID, languageID) 229 except KeyError: 230 return False 231 232 def IsValidTypeAndProperty(self, typeName, propertyName, languageID = None): 233 IS_INVALID_TYPE = 0 234 IS_INVALID_PROPERTY = 1 235 IS_VALID_TYPE_AND_PROPERTY = 2 236 result = None 237 languageID = util.StandardizedLanguageIDOrDefault(languageID) 238 foundType = self.languageTypesWithProperties.get(typeName, None) 239 if foundType is not None: 240 foundPropertyList = foundType.get(languageID, None) 241 if foundPropertyList is not None: 242 if propertyName in foundPropertyList: 243 result = IS_VALID_TYPE_AND_PROPERTY 244 else: 245 result = IS_INVALID_PROPERTY 246 else: 247 result = IS_INVALID_PROPERTY 248 else: 249 result = IS_INVALID_TYPE 250 if result == IS_INVALID_PROPERTY: 251 log.error(u"'%s' is not a valid property for '%s' in language '%s'.", propertyName, typeName, languageID) 252 elif result == IS_INVALID_TYPE: 253 log.error(u"'%s' is not a valid type; cannot retrieve properties for it.", typeName) 254 elif result is None: 255 log.error(u'IsValidTypeAndProperty wasnt able to determine if type and property were valid: %s, %s', typeName, propertyName) 256 return result == IS_VALID_TYPE_AND_PROPERTY 257 258 def GetMetaData(self, messageID, propertyName, languageID = None): 259 languageID = util.StandardizedLanguageIDOrDefault(languageID) 260 propertyString = self._GetMetaData(messageID, propertyName, languageID) 261 if propertyString is not None: 262 return uiutil.PrepareLocalizationSafeString(propertyString, messageID=messageID) 263 log.error(u'a non existent property was requested. messageID,propertyName,languageID : %s,%s,%s', messageID, propertyName, languageID) 264 return u'[no property:%s,%s]' % (messageID, propertyName) 265 266 def GetPlaceholderLabel(self, englishText, **kwargs): 267 tags = parser._Tokenize(englishText) 268 parsedText = eveLocalization.Parse(englishText, locconst.LOCALE_SHORT_ENGLISH, tags, **kwargs) 269 log.warning(u'Placeholder label (%s) needs to be replaced.', englishText) 270 return '!_%s_!' % parsedText 271 272 def _DecideLanguage(self): 273 prefsValue = prefs.GetValue('languageID', None) 274 if prefsValue: 275 log.info(u'prefs language=%r', prefsValue) 276 comfylang = langutils.any_to_comfy_language(prefsValue) 277 if langutils.is_client_valid(comfylang): 278 return comfylang 279 log.warning(u'prefs language invalid=%r', prefsValue) 280 else: 281 log.info(u'no language in prefs') 282 startupValue = self.GetLanguageStartupArg() 283 if startupValue: 284 log.info(u'startup args language=%r', startupValue) 285 comfylang = langutils.any_to_comfy_language(startupValue) 286 if langutils.is_client_valid(comfylang): 287 return comfylang 288 log.warning(u'startup args language invalid=%r', startupValue) 289 else: 290 log.info(u'no language in startup args') 291 log.info(u'using default language=%r', langutils.DEFAULT_CLIENT_LANGUAGE.mls_language_id()) 292 return langutils.DEFAULT_CLIENT_LANGUAGE 293 294 def _ValidateAndRepairPrefs(self): 295 if boot.role == 'client': 296 comfylang = self._DecideLanguage() 297 if comfylang.mls_language_id() != prefs.GetValue('languageID', None): 298 prefs.languageID = comfylang.mls_language_id() 299 blue.os.languageID = comfylang.mls_language_id() 300 301 @telemetry.ZONE_METHOD 302 def _ReadMainLocalizationData(self, unPickledObject, dataType): 303 if unPickledObject and self.FIELD_LABELS in unPickledObject: 304 labelsDict = unPickledObject[self.FIELD_LABELS] 305 for aMessageID in labelsDict: 306 dataRow = labelsDict[aMessageID] 307 pathAndLabelKey = None 308 if dataRow[locconst.PICKLE_LABELS_FULLPATH]: 309 aFullPath = dataRow[locconst.PICKLE_LABELS_FULLPATH] 310 pathAndLabelKey = '/'.join([aFullPath, dataRow[locconst.PICKLE_LABELS_LABEL]]) 311 else: 312 pathAndLabelKey = dataRow[locconst.PICKLE_LABELS_LABEL] 313 self.languageLabels[pathAndLabelKey.encode('ascii')] = aMessageID 314 315 else: 316 log.error(u"didn't find 'labels' in the unpickled object.") 317 return [] 318 langList = [] 319 if self.FIELD_LANGUAGES in unPickledObject: 320 if isinstance(unPickledObject[self.FIELD_LANGUAGES], dict): 321 langList = unPickledObject[self.FIELD_LANGUAGES].keys() 322 else: 323 langList = unPickledObject[self.FIELD_LANGUAGES] 324 langList = filter(self._IsValidLanguage, langList) 325 self.languagesDefined.update(langList) 326 else: 327 log.error(u"didn't find 'languages' in the unpickled object") 328 return [] 329 if self.FIELD_TYPES in unPickledObject: 330 self.languageTypesWithProperties.update(unPickledObject[self.FIELD_TYPES]) 331 else: 332 log.error(u"didn't find 'types' in the unpickled object") 333 return [] 334 return langList 335 336 def _IsValidLanguage(self, languageID): 337 if boot.role == 'client': 338 if boot.region == 'optic' and langutils.is_equal(languageID, langutils.LANG_EN): 339 return True 340 return langutils.is_client_valid(languageID) 341 else: 342 return True 343 344 @telemetry.ZONE_METHOD 345 def _ReadLanguageLocalizationData(self, aLangCode, unPickledObject, dataType): 346 try: 347 log.info(u'Loading all message data for language %r', aLangCode) 348 eveLocalization.LoadMessageData(*unPickledObject) 349 except: 350 logmodule.LogException() 351 for x, y in unPickledObject[1].iteritems(): 352 if y[2] is not None and not isinstance(y[2], dict): 353 log.error(u'%s: %r', x, y) 354 355 return True 356 357 @telemetry.ZONE_METHOD 358 def UsePrimaryLanguageText(self): 359 if self._importantNameSetting is None: 360 if internalUtil.GetLanguageID() == self._primaryLanguageID: 361 self._importantNameSetting = 0 362 else: 363 self._importantNameSetting = bilingualSettings.GetValue('localizationImportantNames') 364 return self._importantNameSetting == bilingualSettings.IMPORTANT_NAME_ENGLISH 365 366 def HighlightImportant(self): 367 if self._highlightImportantSetting is None: 368 if internalUtil.GetLanguageID() == self._primaryLanguageID: 369 self._highlightImportantSetting = self._QaTooltipOverride() 370 else: 371 self._highlightImportantSetting = bilingualSettings.GetValue('localizationHighlightImportant') 372 return self._highlightImportantSetting 373 374 def UseImportantTooltip(self): 375 if self._languageTooltipSetting is None: 376 if internalUtil.GetLanguageID() == self._primaryLanguageID: 377 self._languageTooltipSetting = self._QaTooltipOverride() 378 else: 379 self._languageTooltipSetting = bilingualSettings.GetValue('languageTooltip') 380 return self._languageTooltipSetting 381 382 def _QaTooltipOverride(self): 383 if self._qaTooltipOverride is None: 384 if internalUtil.GetLanguageID() == self._primaryLanguageID: 385 self._qaTooltipOverride = qaSettings.GetValue('simulateTooltip') 386 else: 387 self._qaTooltipOverride 388 return self._qaTooltipOverride 389 390 def ClearImportantNameSetting(self): 391 self._importantNameSetting = None 392 self._highlightImportantSetting = None 393 self._languageTooltipSetting = None 394 self._qaTooltipOverride = None 395 cfg.ReloadLocalizedNames() 396 397 def _InitializeInternalVariables(self): 398 self.hashDataDictionary = {} 399 400 @telemetry.ZONE_METHOD 401 def _InitializeTextData(self): 402 self.languagesDefined = set() 403 self.languageLabels = {} 404 self.importantMessages = {} 405 self.languageTypesWithProperties = {} 406 self._primaryLanguageID = None 407 self._secondaryLanguageID = None 408 409 @telemetry.ZONE_METHOD 410 def UpdateTextCache(self, messagePerLanguage, metaDataPerLanguage, labelsDict): 411 for language in messagePerLanguage: 412 log.info(u'Preparing to update internal text and label cache. The sizes of new data dictionaries are: %s, %s', len(messagePerLanguage.get(language, {})), len(metaDataPerLanguage.get(language, {}))) 413 if eveLocalization.HasLanguage(language): 414 newData = {} 415 for messageID, text in messagePerLanguage.get(language, {}).iteritems(): 416 try: 417 metaData = None 418 ignore, metaData, ignore = eveLocalization.GetMessageDataByID(messageID, language) 419 except KeyError: 420 sys.exc_clear() 421 422 try: 423 newData[messageID] = (text, metaData, parser._Tokenize(text)) 424 except: 425 logmodule.LogException() 426 continue 427 428 try: 429 eveLocalization.LoadMessageData(language, newData) 430 except: 431 logmodule.LogException() 432 continue 433 434 for language in metaDataPerLanguage: 435 if eveLocalization.HasLanguage(language): 436 newData = {} 437 for messageID, metaData in metaDataPerLanguage.get(language, {}).iteritems(): 438 try: 439 text, ignore, tokens = eveLocalization.GetMessageDataByID(messageID, language) 440 newData[messageID] = (text, metaData, tokens) 441 except KeyError: 442 sys.exc_clear() 443 444 try: 445 eveLocalization.LoadMessageData(language, newData) 446 except: 447 logmodule.LogException() 448 continue 449 450 log.info(u'Updating label cache. New data size is %s', len(labelsDict)) 451 for label, messageID in labelsDict.iteritems(): 452 self.languageLabels[label.encode('ascii')] = messageID 453 454 def GetHashDataDictionary(self): 455 return self.hashDataDictionary 456 457 @telemetry.ZONE_METHOD 458 def _ReadLocalizationMainPickle(self, pickleName, dataType): 459 unPickledObject = self._LoadPickleData(pickleName, dataType) 460 if unPickledObject is None: 461 return [] 462 supportedLanguages = self._ReadMainLocalizationData(unPickledObject, dataType) 463 if not supportedLanguages: 464 log.error(u'Error reading main pickle file %s', pickleName) 465 del unPickledObject 466 return supportedLanguages 467 468 def _GetPrimaryLanguage(self): 469 return locconst.LOCALE_SHORT_ENGLISH 470 471 def IsPrimaryLanguage(self, languageID): 472 languageID = util.StandardizedLanguageIDOrDefault(languageID) 473 return languageID == self._GetPrimaryLanguage() 474 475 @telemetry.ZONE_METHOD 476 def _ReadLocalizationLanguagePickles(self, prefix, supportedLanguages, dataType): 477 primaryLanguage = self._GetPrimaryLanguage() 478 if boot.role == 'client': 479 prefsLanguage = util.StandardizedLanguageIDOrDefault(prefs.GetValue('languageID', None)) 480 if prefsLanguage != primaryLanguage and prefsLanguage in supportedLanguages: 481 self._secondaryLanguageID = prefsLanguage 482 if not (self.LoadPrimaryLanguage(prefix, dataType) and self._LoadLanguagePickle(prefix, prefsLanguage, dataType)): 483 return False 484 else: 485 return self.LoadPrimaryLanguage(prefix, dataType) 486 elif boot.role == 'server' or boot.role == 'proxy': 487 if not self.LoadPrimaryLanguage(prefix, dataType): 488 return False 489 for aLangCode in supportedLanguages: 490 if aLangCode != self._primaryLanguageID and not self._LoadLanguagePickle(prefix, aLangCode, dataType): 491 return False 492 493 return True 494 495 @telemetry.ZONE_METHOD 496 def _LoadLanguagePickle(self, prefix, languageID, dataType): 497 unPickledObject = self._LoadPickleData(prefix + languageID + self.PICKLE_EXT, dataType) 498 if unPickledObject == None: 499 return False 500 readStatus = self._ReadLanguageLocalizationData(languageID, unPickledObject, dataType) 501 del unPickledObject 502 return readStatus 503 504 @telemetry.ZONE_METHOD 505 def _LoadPickleData(self, pickleName, dataType): 506 pickleFile = blue.ResFile() 507 if not pickleFile.Open(pickleName): 508 log.error(u'Could not load pickle file. %s appears to be missing. The localization module will not be able to access labels or texts.', pickleName) 509 return None 510 pickledData = pickleFile.Read() 511 if not pickledData: 512 pickleFile.Close() 513 del pickleFile 514 log.error(u'Could not read pickle data from file. %s may be corrupt. The localization module will not be able to access labels or texts.', pickleName) 515 return None 516 blue.statistics.EnterZone('pickle.loads') 517 unPickledObject = pickle.loads(pickledData) 518 blue.statistics.LeaveZone() 519 520 @telemetry.ZONE_FUNCTION 521 def md5ForFile(fin, block_size = 1048576): 522 md5 = hashlib.md5() 523 while True: 524 data = fin.read(block_size) 525 if not data: 526 break 527 md5.update(data) 528 529 return md5.digest() 530 531 self.hashDataDictionary[pickleName] = md5ForFile(pickleFile) 532 pickleFile.Close() 533 del pickleFile 534 del pickledData 535 return unPickledObject 536 537 def _GetRawByMessageID(self, messageID, languageID = None, **kwargs): 538 languageID = util.StandardizedLanguageIDOrDefault(languageID) 539 try: 540 return eveLocalization.GetRawByMessageID(messageID, languageID) 541 except: 542 return u'%s %s]' % (UNRESOLVED_LOC_MARKER, messageID)