binaryRepresenter.py (15811B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\fsd\schemas\binaryRepresenter.py 2 from __future__ import absolute_import 3 import ctypes 4 import cStringIO 5 from . import schemaOptimizer 6 from .path import FsdDataPathObject 7 from . import validator 8 import struct 9 import cPickle 10 from .nestedIndexedOffsetData import IndexedOffsetData 11 uint64 = struct.Struct('Q') 12 uint32 = struct.Struct('I') 13 int32 = struct.Struct('i') 14 vector2_float = struct.Struct('ff') 15 vector2_double = struct.Struct('dd') 16 vector3_float = struct.Struct('fff') 17 vector3_double = struct.Struct('ddd') 18 vector4_float = struct.Struct('ffff') 19 vector4_double = struct.Struct('dddd') 20 21 def StringAsBinaryStringWithIndexedOffsetData(s, schema, path): 22 return (uint32.pack(len(s)) + s, IndexedOffsetData()) 23 24 25 def UnicodeStringAsBinaryStringWithIndexedOffsetData(s, schema, path): 26 encodedString = s.encode('utf-8') 27 return StringAsBinaryStringWithIndexedOffsetData(encodedString, schema, path) 28 29 30 def IntAsBinaryStringWithIndexedOffsetData(i, schema, path): 31 intType = int32 32 if 'min' in schema and schema['min'] >= 0 or 'exclusiveMin' in schema and schema['exclusiveMin'] >= -1: 33 intType = uint32 34 return (intType.pack(i), IndexedOffsetData()) 35 36 37 def FloatAsBinaryStringWithIndexedOffsetData(f, schema, path): 38 if schema.get('precision', 'single') == 'double': 39 t = ctypes.c_double 40 else: 41 t = ctypes.c_float 42 r = t(f) 43 return (ctypes.string_at(ctypes.addressof(r), ctypes.sizeof(r)), IndexedOffsetData()) 44 45 46 def Vector2AsBinaryStringWithIndexedOffsetData(v, schema, path): 47 if schema.get('precision', 'single') == 'double': 48 t = vector2_double 49 else: 50 t = vector2_float 51 return (t.pack(*v), IndexedOffsetData()) 52 53 54 def Vector3AsBinaryStringWithIndexedOffsetData(v, schema, path): 55 if schema.get('precision', 'single') == 'double': 56 t = vector3_double 57 else: 58 t = vector3_float 59 return (t.pack(*v), IndexedOffsetData()) 60 61 62 def Vector4AsBinaryStringWithIndexedOffsetData(v, schema, path): 63 if schema.get('precision', 'single') == 'double': 64 t = vector4_double 65 else: 66 t = vector4_float 67 return (t.pack(*v), IndexedOffsetData()) 68 69 70 def BoolAsBinaryStringWithIndexedOffsetData(v, schema, path): 71 r = ctypes.c_ubyte(255 if v else 0) 72 return (ctypes.string_at(ctypes.addressof(r), ctypes.sizeof(r)), IndexedOffsetData()) 73 74 75 def EnumAsBinaryStringWithIndexedOffsetData(e, schema, path): 76 maxEnumValue = max(schema['values'].values()) 77 enumType = schemaOptimizer.GetLargeEnoughUnsignedTypeForMaxValue(maxEnumValue) 78 r = enumType(schema['values'][e]) 79 return (ctypes.string_at(ctypes.addressof(r), ctypes.sizeof(r)), IndexedOffsetData()) 80 81 82 def ListAsBinaryStringWithIndexedOffsetData(l, schema, path): 83 representation = cStringIO.StringIO() 84 listLength = len(l) 85 indexedOffsetDataForListItems = IndexedOffsetData() 86 if 'length' not in schema: 87 representation.write(uint32.pack(listLength)) 88 if 'fixedItemSize' in schema: 89 for i, item in enumerate(l): 90 b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(item, schema['itemTypes'], FsdDataPathObject('[%s]' % i, parent=path)) 91 representation.write(b) 92 93 else: 94 baseOffset = ctypes.sizeof(ctypes.c_uint32) * (listLength + 1) 95 offset = baseOffset 96 offsetArray = (ctypes.c_uint32 * listLength)() 97 listData = cStringIO.StringIO() 98 for i, item in enumerate(l): 99 b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(item, schema['itemTypes'], FsdDataPathObject('[%s]' % i, parent=path)) 100 offsetArray[i] = offset 101 offset += len(b) 102 listData.write(b) 103 if not indexedOffsetData.isEmpty(): 104 indexedOffsetData.AddOffset(offsetArray[i]) 105 indexedOffsetDataForListItems.AddNestedIndexedOffsetData(indexedOffsetData) 106 107 offsetHeader = ctypes.string_at(ctypes.addressof(offsetArray), ctypes.sizeof(offsetArray)) 108 representation.write(offsetHeader) 109 representation.write(listData.getvalue()) 110 return (representation.getvalue(), indexedOffsetDataForListItems) 111 112 113 def DictAsBinaryStringWithIndexedOffsetData(d, schema, path): 114 keySchema = schema['keyFooter'] 115 sortedKeyList = [ k for k in sorted(d.keys()) ] 116 offsets = [] 117 indexedOffsetData = IndexedOffsetData() 118 dataRepresentation = cStringIO.StringIO() 119 offsetToData = 4 120 includeDataItemSizeInKeyList = 'size' in keySchema['itemTypes']['attributes'] 121 for key in sortedKeyList: 122 b, nestedOffsetData = RepresentAsBinaryWithIndexedOffsetData(d[key], schema['valueTypes'], FsdDataPathObject('[%s]' % key, parent=path)) 123 offset = dataRepresentation.tell() 124 footerData = {'key': key, 125 'offset': offset} 126 if not nestedOffsetData.isEmpty(): 127 nestedOffsetData.AddOffset(offset + offsetToData) 128 indexedOffsetData.AddNestedIndexedOffsetData(nestedOffsetData) 129 if includeDataItemSizeInKeyList: 130 footerData['size'] = len(b) 131 offsets.append(footerData) 132 dataRepresentation.write(b) 133 134 offsetTable = RepresentAsBinary(offsets, keySchema, path) 135 dictFooter = offsetTable + uint32.pack(len(offsetTable)) 136 binaryData = dataRepresentation.getvalue() 137 if 'indexBy' in schema: 138 for offset in offsets: 139 indexedOffsetData.AddKeyOffsetSizeAndPathToNestedIndexId(offset['key'], offset['offset'], offset['size'], path, schema['nestedIndexId']) 140 141 elif schema.get('multiIndex', False): 142 offsetToIndexedOffset = 4 + len(binaryData) 143 nestedOffsets = RepresentIndexedOffsetDataAsBinary(indexedOffsetData, schema, offsetToIndexedOffset) 144 dictFooter = nestedOffsets + dictFooter 145 return (uint32.pack(len(binaryData + dictFooter)) + binaryData + dictFooter, indexedOffsetData) 146 147 148 def RepresentIndexedOffsetDataAsBinary(indexedOffsetData, schema, offsetToAttribute): 149 indexableSchemas = schema['indexableSchemas'] 150 offsetToIndexedOffsetData = offsetToAttribute 151 binaryOffsetData = cStringIO.StringIO() 152 offsetLookupTable = {} 153 flattenedIndexedOffsetData = indexedOffsetData.Flatten() 154 for nestedIndex, indexableSchema in indexableSchemas.iteritems(): 155 if nestedIndex not in flattenedIndexedOffsetData: 156 recreatedOffsetData = [] 157 else: 158 recreatedOffsetData = [ {'key': key, 159 'offset': offset, 160 'size': size} for key, offset, size, path in flattenedIndexedOffsetData[nestedIndex] ] 161 b = RepresentAsBinary(recreatedOffsetData, indexableSchema['keyFooter'], None) 162 binaryOffsetData.write(b) 163 offsetLookupTable[nestedIndex] = {'offset': offsetToIndexedOffsetData, 164 'size': len(b)} 165 offsetToIndexedOffsetData += len(b) 166 167 builtOffsetLookupTable = RepresentAsBinary(offsetLookupTable, schema['subIndexOffsetLookup'], None) 168 return binaryOffsetData.getvalue() + builtOffsetLookupTable + uint32.pack(len(builtOffsetLookupTable)) 169 170 171 def StreamSortedIndexToFile(fileObjectOut, keyValueAndMaybeIndexedOffsetData, totalCount, schema, embedSchema = False): 172 keyFooterSchema = schema['keyFooter'] 173 if 'fixedItemSize' not in keyFooterSchema: 174 raise ValueError('keys must have a known size to stream the index') 175 startingOffset = 0 176 if embedSchema: 177 schemaToRepresent = {} 178 schemaToRepresent.update(schema) 179 if schema['valueTypes']['type'] == 'binary' and 'schema' in schema['valueTypes']: 180 schemaToRepresent['valueTypes'] = schema['valueTypes']['schema'] 181 fileObjectOut.write(RepresentSchemaAsBinary(schemaToRepresent)) 182 startingOffset = fileObjectOut.tell() 183 fileObjectOut.write(uint32.pack(0)) 184 footerList = [] 185 path = FsdDataPathObject('[Stream]') 186 offset = 0 187 lengthOfFileObjectHeader = 4 188 indexedOffsetDataForStream = IndexedOffsetData() 189 for d in keyValueAndMaybeIndexedOffsetData: 190 key, value = d[:2] 191 indexedOffsetDataContainedInKeyValueSet = len(d) == 3 192 prebuiltIndexedOffsetData = d[2] if indexedOffsetDataContainedInKeyValueSet else IndexedOffsetData() 193 currentFilePos = fileObjectOut.tell() 194 pathToValues = FsdDataPathObject('[%s]' % key, path) 195 b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(value, schema['valueTypes'], pathToValues) 196 fileObjectOut.write(b) 197 footer = {'key': key, 198 'offset': offset} 199 if schema.get('buildIndex', False): 200 footer['size'] = len(b) 201 footerList.append(footer) 202 if not prebuiltIndexedOffsetData.isEmpty(): 203 prebuiltIndexedOffsetData.AddOffset(offset + lengthOfFileObjectHeader) 204 indexedOffsetDataForStream.AddNestedIndexedOffsetData(prebuiltIndexedOffsetData) 205 if not indexedOffsetData.isEmpty(): 206 indexedOffsetData.AddOffset(offset + lengthOfFileObjectHeader) 207 indexedOffsetDataForStream.AddNestedIndexedOffsetData(indexedOffsetData) 208 offset += fileObjectOut.tell() - currentFilePos 209 210 if 'multiIndex' in schema: 211 multiIndexAttributes = RepresentIndexedOffsetDataAsBinary(indexedOffsetDataForStream, schema, fileObjectOut.tell() - startingOffset) 212 fileObjectOut.write(multiIndexAttributes) 213 primaryIndexAsBinary = RepresentAsBinary(footerList, keyFooterSchema, FsdDataPathObject('<Footer>', path)) 214 fileObjectOut.write(primaryIndexAsBinary) 215 fileObjectOut.write(uint32.pack(len(primaryIndexAsBinary))) 216 totalFileObjectSize = fileObjectOut.tell() - startingOffset 217 fileObjectOut.seek(startingOffset) 218 fileObjectOut.write(uint32.pack(totalFileObjectSize - 4)) 219 220 221 def ObjectAsBinaryStringWithIndexedOffsetData(o, schema, path): 222 representation = cStringIO.StringIO() 223 attributesByOffset = [ (off, attr) for attr, off in schema['constantAttributeOffsets'].iteritems() ] 224 attributesByOffset.sort() 225 indexedOffsetDataForObjectAttributes = IndexedOffsetData() 226 for offset, attr in attributesByOffset: 227 b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(o[attr], schema['attributes'][attr], FsdDataPathObject('.%s' % attr, parent=path)) 228 representation.write(b) 229 if not indexedOffsetData.isEmpty(): 230 indexedOffsetData.AddOffset(offset) 231 indexedOffsetDataForObjectAttributes.AddNestedIndexedOffsetData(indexedOffsetData) 232 233 if 'size' in schema: 234 return (representation.getvalue(), indexedOffsetDataForObjectAttributes) 235 variableAndOptionalAttributeOrder = schema['attributesWithVariableOffsets'][:] 236 optionalAttributesField = 0 237 for i in schema['optionalValueLookups']: 238 if i in o: 239 optionalAttributesField |= schema['optionalValueLookups'][i] 240 else: 241 variableAndOptionalAttributeOrder.remove(i) 242 243 offsets = (ctypes.c_uint32 * len(variableAndOptionalAttributeOrder))() 244 attributesWithOffsets = cStringIO.StringIO() 245 for i, attr in enumerate(variableAndOptionalAttributeOrder): 246 offsets[i] = attributesWithOffsets.tell() 247 b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(o[attr], schema['attributes'][attr], FsdDataPathObject('.%s' % attr, parent=path)) 248 attributesWithOffsets.write(b) 249 if not indexedOffsetData.isEmpty(): 250 indexedOffsetData.AddOffset(offsets[i] + representation.tell()) 251 indexedOffsetDataForObjectAttributes.AddNestedIndexedOffsetData(indexedOffsetData) 252 253 offsetToListData = len(uint64.pack(optionalAttributesField)) + len(ctypes.string_at(ctypes.addressof(offsets), ctypes.sizeof(offsets))) 254 indexedOffsetDataForObjectAttributes.AddOffset(offsetToListData) 255 representation.write(uint64.pack(optionalAttributesField)) 256 representation.write(ctypes.string_at(ctypes.addressof(offsets), ctypes.sizeof(offsets))) 257 representation.write(attributesWithOffsets.getvalue()) 258 return (representation.getvalue(), indexedOffsetDataForObjectAttributes) 259 260 261 def BinaryStringAsBinaryStringWithIndexedOffsetData(o, schema, path): 262 return (o, IndexedOffsetData()) 263 264 265 def UnionAsBinaryStringWithIndexedOffsetData(o, schema, path): 266 representation = cStringIO.StringIO() 267 for idx, possibleSchemaType in enumerate(schema['optionTypes']): 268 if len(validator.Validate(possibleSchemaType, o)) == 0: 269 binaryIndex = uint32.pack(idx) 270 representation.write(binaryIndex) 271 itemRepresentation, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(o, possibleSchemaType, FsdDataPathObject('<%s>' % possibleSchemaType['type'], parent=path)) 272 if not indexedOffsetData.isEmpty(): 273 indexedOffsetData.AddOffset(len(binaryIndex)) 274 representation.write(itemRepresentation) 275 return (representation.getvalue(), indexedOffsetData) 276 277 raise ValueError('Could not represent %s as any type in union' % str(o)) 278 279 280 builtInRepresenters = {'enum': EnumAsBinaryStringWithIndexedOffsetData, 281 'bool': BoolAsBinaryStringWithIndexedOffsetData, 282 'vector4': Vector4AsBinaryStringWithIndexedOffsetData, 283 'vector3': Vector3AsBinaryStringWithIndexedOffsetData, 284 'vector2': Vector2AsBinaryStringWithIndexedOffsetData, 285 'float': FloatAsBinaryStringWithIndexedOffsetData, 286 'int': IntAsBinaryStringWithIndexedOffsetData, 287 'typeID': IntAsBinaryStringWithIndexedOffsetData, 288 'localizationID': IntAsBinaryStringWithIndexedOffsetData, 289 'string': StringAsBinaryStringWithIndexedOffsetData, 290 'resPath': StringAsBinaryStringWithIndexedOffsetData, 291 'list': ListAsBinaryStringWithIndexedOffsetData, 292 'dict': DictAsBinaryStringWithIndexedOffsetData, 293 'object': ObjectAsBinaryStringWithIndexedOffsetData, 294 'binary': BinaryStringAsBinaryStringWithIndexedOffsetData, 295 'union': UnionAsBinaryStringWithIndexedOffsetData, 296 'unicode': UnicodeStringAsBinaryStringWithIndexedOffsetData, 297 'color': Vector4AsBinaryStringWithIndexedOffsetData, 298 'npcTag': IntAsBinaryStringWithIndexedOffsetData, 299 'deploymentType': IntAsBinaryStringWithIndexedOffsetData, 300 'npcEnemyFleetTypeID': IntAsBinaryStringWithIndexedOffsetData, 301 'groupBehaviorTreeID': IntAsBinaryStringWithIndexedOffsetData, 302 'npcCorporationID': IntAsBinaryStringWithIndexedOffsetData, 303 'spawnTableID': IntAsBinaryStringWithIndexedOffsetData, 304 'npcFleetCounterTableID': IntAsBinaryStringWithIndexedOffsetData, 305 'dungeonID': IntAsBinaryStringWithIndexedOffsetData, 306 'typeListID': IntAsBinaryStringWithIndexedOffsetData, 307 'npcFleetTypeID': IntAsBinaryStringWithIndexedOffsetData, 308 'metaGroupID': IntAsBinaryStringWithIndexedOffsetData, 309 'fsdReference': IntAsBinaryStringWithIndexedOffsetData, 310 'raceID': IntAsBinaryStringWithIndexedOffsetData, 311 'marketGroupID': IntAsBinaryStringWithIndexedOffsetData, 312 'ShipGroupID': IntAsBinaryStringWithIndexedOffsetData, 313 'certificateTemplateID': IntAsBinaryStringWithIndexedOffsetData, 314 'factionID': IntAsBinaryStringWithIndexedOffsetData} 315 316 def RepresentAsBinaryWithIndexedOffsetData(o, schema, path): 317 schemaType = schema.get('type') 318 if schemaType in builtInRepresenters: 319 data, indexedDictOffsets = builtInRepresenters[schemaType](o, schema, path) 320 return (data, indexedDictOffsets) 321 raise NotImplementedError("Type '%s' does not have a binary representation implementation" % schemaType) 322 323 324 def RepresentAsBinary(o, schema, path = None): 325 if path is None: 326 path = FsdDataPathObject('<root>') 327 binaryData = RepresentAsBinaryWithIndexedOffsetData(o, schema, path)[0] 328 return binaryData 329 330 331 def RepresentSchemaAsBinary(schema): 332 binarySchema = cPickle.dumps(schema) 333 return uint32.pack(len(binarySchema)) + binarySchema 334 335 336 def RepresentAsBinaryWithEmbeddedSchema(o, schema, path = None): 337 return RepresentSchemaAsBinary(schema) + RepresentAsBinary(o, schema, path)