schemaOptimizer.py (16566B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\fsd\schemas\schemaOptimizer.py 2 from __future__ import print_function 3 from __future__ import absolute_import 4 import copy 5 import collections 6 import ctypes 7 import yaml 8 from . import validator 9 from . import persistence 10 import os 11 from . import schemaCompare 12 builtInCTypesObjectTypes = {'typeID': ctypes.c_uint32, 13 'localizationID': ctypes.c_uint32, 14 'bool': ctypes.c_ubyte} 15 subIndexOffsetLookupSchema = {'type': 'dict', 16 'keyTypes': {'type': 'int', 17 'min': 0}, 18 'valueTypes': {'type': 'object', 19 'attributes': {'offset': {'type': 'int', 20 'min': 0}, 21 'size': {'type': 'int', 22 'min': 0}}}} 23 FLOAT_PRECISION_DEFAULT = 'single' 24 fixedSizeTypes = set(['float', 25 'int', 26 'typeID', 27 'localizationID', 28 'vector2', 29 'vector3', 30 'vector4', 31 'enum', 32 'bool']) 33 34 class IndexingNotSupported(Exception): 35 36 def __init__(self, schemaType): 37 self.schemaType = schemaType 38 39 def __str__(self): 40 return 'Schema of type: "%s" does not support the "indexBy" attribute' % self.schemaType 41 42 43 def GetKeyHeaderSchema(keySchema): 44 keyHeaderSchema = {'type': 'list', 45 'itemTypes': {'type': 'object', 46 'attributes': collections.OrderedDict()}} 47 attrs = keyHeaderSchema['itemTypes']['attributes'] 48 attrs['key'] = keySchema 49 attrs['offset'] = {'type': 'int', 50 'min': 0} 51 return keyHeaderSchema 52 53 54 def GetKeyHeaderSchemaWithSize(keySchema): 55 keyHeaderSchema = GetKeyHeaderSchema(keySchema) 56 keyHeaderSchema['itemTypes']['attributes']['size'] = {'type': 'int', 57 'min': 0} 58 return keyHeaderSchema 59 60 61 def IsUsedInTarget(schema, target): 62 usage = schema.get('usage', None) 63 if usage is None or usage == target or target == validator.EDITOR: 64 return True 65 else: 66 return False 67 68 69 def GetLargeEnoughUnsignedTypeForMaxValue(i): 70 if i <= 255: 71 return ctypes.c_ubyte 72 elif i <= 65536: 73 return ctypes.c_uint16 74 else: 75 return ctypes.c_uint32 76 77 78 def GetAttributesForTarget(schema, target): 79 orderedAttributeList = [] 80 attributes = schema.get('attributes') 81 for attr in attributes: 82 if IsUsedInTarget(attributes[attr], target): 83 orderedAttributeList.append(attr) 84 85 return orderedAttributeList 86 87 88 def GetOptionalAttributesForTarget(schema, target): 89 return list(OptionalAttributeIterator(schema, target)) 90 91 92 def IsAttributeOptional(schema, attributeName): 93 return schema['attributes'][attributeName].get('isOptional', False) 94 95 96 def OptionalAttributeIterator(schema, target): 97 attributes = GetAttributesForTarget(schema, target) 98 for attr in attributes: 99 if IsAttributeOptional(schema, attr): 100 yield attr 101 102 103 def OptionalAndVariableSizedAttributeIterator(schema, target, pathRoot): 104 attributes = GetAttributesForTarget(schema, target) 105 for attr in attributes: 106 if IsAttributeOptional(schema, attr): 107 yield attr 108 elif not IsFixedSize(schema['attributes'][attr], target, pathRoot): 109 yield attr 110 111 112 def FixedSizeAttributeIterator(schema, target, pathRoot): 113 attributes = GetAttributesForTarget(schema, target) 114 for attr in attributes: 115 if IsAttributeOptional(schema, attr): 116 continue 117 elif not IsFixedSize(schema['attributes'][attr], target, pathRoot): 118 continue 119 yield attr 120 121 122 def IsFixedSize(schema, target, pathRoot): 123 if 'schemaReference' in schema: 124 schema = LoadReferencedSchema(schema['schemaReference'], pathRoot) 125 return IsFixedSize(schema, target, pathRoot) 126 schemaType = schema['type'] 127 if schemaType in fixedSizeTypes: 128 return True 129 if schemaType == 'object': 130 attributes = schema['attributes'] 131 for attr in attributes: 132 if not IsUsedInTarget(attributes[attr], target): 133 continue 134 if attributes[attr].get('isOptional', False) or not IsFixedSize(attributes[attr], target, pathRoot): 135 return False 136 137 return True 138 return False 139 140 141 def GetIndexableSchemasGenerator(schema): 142 stack = [schema] 143 while len(stack) > 0: 144 currentSchema = stack.pop() 145 currentSchemaType = currentSchema['type'] 146 for key in currentSchema: 147 if key == 'indexBy': 148 if currentSchemaType != 'dict': 149 raise IndexingNotSupported(currentSchemaType) 150 yield currentSchema 151 elif key in ('valueTypes', 'itemTypes'): 152 stack.append(currentSchema[key]) 153 elif key == 'attributes': 154 for attribute in currentSchema['attributes']: 155 stack.append(currentSchema['attributes'][attribute]) 156 157 elif key == 'optionTypes': 158 for optionSchema in currentSchema['optionTypes']: 159 stack.append(optionSchema) 160 161 elif key == 'schema': 162 stack.append(currentSchema['schema']) 163 164 165 def GetEnumeratedIndexableSchemas(schema): 166 enumeratedIndexableSchemas = {} 167 for index, nestedSchema in enumerate(GetIndexableSchemasGenerator(schema)): 168 enumeratedIndexableSchemas[index] = nestedSchema 169 170 return enumeratedIndexableSchemas 171 172 173 def SetNestedIndexIdInformationToIndexableSchemas(idsToIndexableSchemaLists): 174 for index, indexableSchemaList in idsToIndexableSchemaLists.iteritems(): 175 for indexableSchema in indexableSchemaList: 176 indexableSchema['nestedIndexId'] = index 177 178 179 def GatherEqualIndexableSchemasToUniqueId(enumeratedIndexableSchemas): 180 processedSchemas = [] 181 indexIdToSchemas = {} 182 for index, indexableSchema in enumeratedIndexableSchemas.iteritems(): 183 schemaEqualToAnyProcessedSchema = [ schemaCompare.SchemasEqual(indexableSchema, s) for s in processedSchemas ] 184 if not any(schemaEqualToAnyProcessedSchema): 185 processedSchemas.append(indexableSchema) 186 indexIdToSchemas[index] = [indexableSchema] 187 else: 188 i = schemaEqualToAnyProcessedSchema.index(True) 189 indexIdToSchemas[i].append(indexableSchema) 190 191 return indexIdToSchemas 192 193 194 def GetNestedIndexNamesAsList(schema): 195 indexNames = schema['indexBy'] 196 if not isinstance(schema['indexBy'], list): 197 indexNames = [indexNames] 198 return indexNames 199 200 201 def GetNestedIndexNamesMappedToIndexId(enumeratedIndexableSchemas): 202 indexNameToNestedIndexIds = {} 203 for nestedIndexId, nestedDictSchema in enumeratedIndexableSchemas.iteritems(): 204 indexNames = GetNestedIndexNamesAsList(nestedDictSchema) 205 for indexName in indexNames: 206 if indexName not in indexNameToNestedIndexIds: 207 indexNameToNestedIndexIds[indexName] = [] 208 indexNameToNestedIndexIds[indexName].append(nestedIndexId) 209 210 return indexNameToNestedIndexIds 211 212 213 def SetNestedIndexIdInformationToRootSchema(schema): 214 unOptimizedIdToSchemaLookup = GetEnumeratedIndexableSchemas(schema) 215 idsToIndexableSchemaList = GatherEqualIndexableSchemasToUniqueId(unOptimizedIdToSchemaLookup) 216 SetNestedIndexIdInformationToIndexableSchemas(idsToIndexableSchemaList) 217 optimizedIdToSchemaLookup = {} 218 for s in unOptimizedIdToSchemaLookup.values(): 219 optimizedIdToSchemaLookup[s['nestedIndexId']] = s 220 221 schema['indexableSchemas'] = optimizedIdToSchemaLookup 222 schema['indexNameToIds'] = GetNestedIndexNamesMappedToIndexId(optimizedIdToSchemaLookup) 223 224 225 def OptimizeDictSchema(schema, target, pathRoot): 226 newSchema = {'type': 'dict', 227 'keyTypes': OptimizeSchema(schema['keyTypes'], target, pathRoot), 228 'valueTypes': OptimizeSchema(schema['valueTypes'], target, pathRoot)} 229 if 'isOptional' in schema: 230 newSchema['isOptional'] = schema['isOptional'] 231 requiresSizeLookup = False 232 if schema.get('buildIndex', False) or schema.get('multiIndex', False): 233 newSchema['buildIndex'] = True 234 requiresSizeLookup = True 235 if 'indexBy' in schema: 236 requiresSizeLookup = True 237 newSchema['indexBy'] = schema['indexBy'] 238 if 'multiIndex' in schema: 239 newSchema['multiIndex'] = schema['multiIndex'] 240 newSchema['subIndexOffsetLookup'] = OptimizeSchema(subIndexOffsetLookupSchema, target, pathRoot) 241 SetNestedIndexIdInformationToRootSchema(newSchema) 242 if requiresSizeLookup: 243 keyFooterSchema = GetKeyHeaderSchemaWithSize(schema['keyTypes']) 244 else: 245 keyFooterSchema = GetKeyHeaderSchema(schema['keyTypes']) 246 newSchema['keyFooter'] = OptimizeSchema(keyFooterSchema, target, pathRoot) 247 return newSchema 248 249 250 def OptimizeListSchema(schema, target, pathRoot): 251 newSchema = {'type': 'list', 252 'itemTypes': OptimizeSchema(schema['itemTypes'], target, pathRoot)} 253 if 'size' in newSchema['itemTypes']: 254 newSchema['fixedItemSize'] = newSchema['itemTypes']['size'] 255 if 'length' in schema: 256 newSchema['length'] = schema['length'] 257 if 'isOptional' in schema: 258 newSchema['isOptional'] = schema['isOptional'] 259 return newSchema 260 261 262 def OptimizeObjectSchema(schema, target, pathRoot): 263 newSchema = {'type': 'object'} 264 if 'isOptional' in schema: 265 newSchema['isOptional'] = schema['isOptional'] 266 newSchema['attributes'] = collections.OrderedDict() 267 newSchema['constantAttributeOffsets'] = {} 268 newSchema['attributesWithVariableOffsets'] = [] 269 newSchema['optionalValueLookups'] = {} 270 newSchema['size'] = 0 271 currentOffset = 0 272 optionalAttributeValue = 1 273 attributeSchemas = schema['attributes'] 274 for attributeName in FixedSizeAttributeIterator(schema, target, pathRoot): 275 optimizedAttributeSchema = OptimizeSchema(attributeSchemas[attributeName], target, pathRoot) 276 knownSizeOfAttribute = optimizedAttributeSchema['size'] 277 newSchema['constantAttributeOffsets'][attributeName] = currentOffset 278 newSchema['attributes'][attributeName] = optimizedAttributeSchema 279 currentOffset += knownSizeOfAttribute 280 281 newSchema['endOfFixedSizeData'] = currentOffset 282 newSchema['size'] = currentOffset 283 for attributeName in OptionalAndVariableSizedAttributeIterator(schema, target, pathRoot): 284 if 'size' in newSchema: 285 del newSchema['size'] 286 if IsAttributeOptional(schema, attributeName): 287 newSchema['optionalValueLookups'][attributeName] = optionalAttributeValue 288 optionalAttributeValue <<= 1 289 optimizedAttributeSchema = OptimizeSchema(attributeSchemas[attributeName], target, pathRoot) 290 newSchema['attributes'][attributeName] = optimizedAttributeSchema 291 newSchema['attributesWithVariableOffsets'].append(attributeName) 292 293 if optionalAttributeValue != 1: 294 newSchema['maxBitFieldValue'] = optionalAttributeValue >> 1 295 return newSchema 296 297 298 def OptimizeEnumSchema(schema, target, pathRoot): 299 newSchema = {'type': 'enum'} 300 maxEnumValue = max(schema['values'].values()) 301 newSchema['maxEnumValue'] = maxEnumValue 302 newSchema['size'] = ctypes.sizeof(GetLargeEnoughUnsignedTypeForMaxValue(maxEnumValue)) 303 newSchema['values'] = schema['values'] 304 if 'readEnumValue' in schema: 305 newSchema['readEnumValue'] = schema['readEnumValue'] 306 if 'isOptional' in schema: 307 newSchema['isOptional'] = schema['isOptional'] 308 if 'default' in schema: 309 newSchema['default'] = schema['default'] 310 return newSchema 311 312 313 def OptimizeIntSchema(schema, target, pathRoot): 314 newSchema = {'type': 'int'} 315 intType = ctypes.c_int32 316 if 'min' in schema: 317 newSchema['min'] = schema['min'] 318 if 'exclusiveMin' in schema: 319 newSchema['exclusiveMin'] = schema['exclusiveMin'] 320 if 'max' in schema: 321 newSchema['max'] = schema['max'] 322 if 'exclusiveMax' in schema: 323 newSchema['exclusiveMax'] = schema['exclusiveMax'] 324 newSchema['size'] = ctypes.sizeof(intType) 325 if 'isOptional' in schema: 326 newSchema['isOptional'] = schema['isOptional'] 327 if 'default' in schema: 328 newSchema['default'] = schema['default'] 329 return newSchema 330 331 332 def OptimizeFloatSchema(schema, target, pathRoot): 333 newSchema = {'type': schema['type']} 334 if 'isOptional' in schema: 335 newSchema['isOptional'] = schema['isOptional'] 336 if 'precision' in schema: 337 newSchema['precision'] = schema['precision'] 338 precision = schema.get('precision', FLOAT_PRECISION_DEFAULT) 339 if precision == 'double': 340 newSchema['size'] = ctypes.sizeof(ctypes.c_double) 341 else: 342 newSchema['size'] = ctypes.sizeof(ctypes.c_float) 343 if 'default' in schema: 344 newSchema['default'] = schema['default'] 345 return newSchema 346 347 348 def OptimizeVectorSchema(schema, target, pathRoot): 349 newSchema = {'type': schema['type']} 350 if 'aliases' in schema: 351 newSchema['aliases'] = schema['aliases'] 352 if 'isOptional' in schema: 353 newSchema['isOptional'] = schema['isOptional'] 354 if 'precision' in schema: 355 newSchema['precision'] = schema['precision'] 356 if schema['type'] == 'vector2': 357 itemCount = 2 358 elif schema['type'] == 'vector3': 359 itemCount = 3 360 elif schema['type'] == 'vector4': 361 itemCount = 4 362 precision = schema.get('precision', FLOAT_PRECISION_DEFAULT) 363 if precision == 'double': 364 newSchema['size'] = ctypes.sizeof(ctypes.c_double * itemCount) 365 else: 366 newSchema['size'] = ctypes.sizeof(ctypes.c_float * itemCount) 367 return newSchema 368 369 370 def OptimizeBinarySchema(schema, target, pathRoot): 371 newSchema = {'type': schema['type']} 372 if 'schema' in schema: 373 dataSchema = OptimizeSchema(schema['schema'], target, pathRoot) 374 newSchema['schema'] = dataSchema 375 if 'size' in dataSchema: 376 newSchema['size'] = dataSchema['size'] 377 if 'isOptional' in dataSchema: 378 newSchema['isOptional'] = dataSchema['isOptional'] 379 return newSchema 380 381 382 def OptimizeUnionSchema(schema, target, pathRoot): 383 newSchema = {'type': schema['type'], 384 'optionTypes': []} 385 for unionType in schema['optionTypes']: 386 dataSchema = OptimizeSchema(unionType, target, pathRoot) 387 newSchema['optionTypes'].append(dataSchema) 388 389 return newSchema 390 391 392 builtInSchemaOptimizationFunctions = {'dict': OptimizeDictSchema, 393 'list': OptimizeListSchema, 394 'object': OptimizeObjectSchema, 395 'enum': OptimizeEnumSchema, 396 'int': OptimizeIntSchema, 397 'vector2': OptimizeVectorSchema, 398 'vector3': OptimizeVectorSchema, 399 'vector4': OptimizeVectorSchema, 400 'binary': OptimizeBinarySchema, 401 'float': OptimizeFloatSchema, 402 'union': OptimizeUnionSchema} 403 404 def LoadReferencedSchema(reference, pathRoot): 405 schemaFilePath = reference.get('schemaFile') 406 schemaName = reference.get('schema') 407 print('loading', schemaName, 'from', schemaFilePath) 408 if pathRoot is None: 409 raise Exception('Could not optimize schema without a path root with which to load referenced schemas') 410 with open(os.path.join(pathRoot, schemaFilePath), 'r') as schemaFile: 411 schemas = persistence.LoadSchema(schemaFile)['schemas'] 412 schema = schemas[schemaName] 413 return schema 414 415 416 def OptimizeSchema(schema, target, pathRoot = None): 417 reference = schema.get('schemaReference', None) 418 if reference is not None: 419 schema = LoadReferencedSchema(reference, pathRoot) 420 schemaType = schema.get('type') 421 if schemaType in builtInSchemaOptimizationFunctions: 422 return builtInSchemaOptimizationFunctions[schemaType](schema, target, pathRoot) 423 else: 424 newSchema = {'type': schemaType} 425 if 'isOptional' in schema: 426 newSchema['isOptional'] = schema['isOptional'] 427 if 'default' in schema: 428 newSchema['default'] = schema['default'] 429 if schemaType in builtInCTypesObjectTypes: 430 newSchema['size'] = ctypes.sizeof(builtInCTypesObjectTypes[schemaType]) 431 return newSchema 432 433 434 def orderedDict_representer(dumper, data): 435 mappingSequence = [] 436 n = yaml.MappingNode(u'tag:yaml.org,2002:map', mappingSequence, flow_style=False) 437 dumper.represented_objects[id(data)] = n 438 for item_key, item_value in data.iteritems(): 439 node_key = dumper.represent_data(item_key) 440 node_value = dumper.represent_data(item_value) 441 mappingSequence.append((node_key, node_value)) 442 443 return n 444 445 446 class SafeSchemaRepresenter(yaml.SafeDumper): 447 pass 448 449 450 SafeSchemaRepresenter.add_representer(collections.OrderedDict, orderedDict_representer) 451 452 def PersistOptimizedSchema(schema): 453 return yaml.dump(schema, Dumper=SafeSchemaRepresenter)