binaryLoader.py (8094B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\fsd\schemas\binaryLoader.py 2 from __future__ import absolute_import 3 import cPickle 4 import ctypes 5 from . import persistence 6 import struct 7 from . import loaders 8 from .loaders import objectLoader, dictLoader, listLoader 9 import logging 10 import os 11 from eveprefs import boot 12 from .path import FsdDataPathObject 13 uint32 = struct.Struct('I') 14 log = logging.getLogger(__name__) 15 defaultLoaderFactories = {'float': loaders.FloatFromBinaryString, 16 'vector4': loaders.Vector4FromBinaryString, 17 'color': loaders.Vector4FromBinaryString, 18 'vector3': loaders.Vector3FromBinaryString, 19 'vector2': loaders.Vector2FromBinaryString, 20 'string': loaders.StringFromBinaryString, 21 'resPath': loaders.StringFromBinaryString, 22 'enum': loaders.EnumFromBinaryString, 23 'bool': loaders.BoolFromBinaryString, 24 'int': loaders.IntFromBinaryString, 25 'typeID': loaders.IntFromBinaryString, 26 'localizationID': loaders.IntFromBinaryString, 27 'union': loaders.UnionFromBinaryString, 28 'list': loaders.listLoader.ListFromBinaryString, 29 'object': loaders.objectLoader.ObjectLoader, 30 'dict': loaders.dictLoader.DictLoader, 31 'unicode': loaders.UnicodeStringFromBinaryString, 32 'npcTag': loaders.IntFromBinaryString, 33 'deploymentType': loaders.IntFromBinaryString, 34 'npcEnemyFleetTypeID': loaders.IntFromBinaryString, 35 'groupBehaviorTreeID': loaders.IntFromBinaryString, 36 'npcCorporationID': loaders.IntFromBinaryString, 37 'spawnTableID': loaders.IntFromBinaryString, 38 'npcFleetCounterTableID': loaders.IntFromBinaryString, 39 'dungeonID': loaders.IntFromBinaryString, 40 'typeListID': loaders.IntFromBinaryString, 41 'npcFleetTypeID': loaders.IntFromBinaryString, 42 'metaGroupID': loaders.IntFromBinaryString, 43 'fsdReference': loaders.IntFromBinaryString, 44 'raceID': loaders.IntFromBinaryString, 45 'marketGroupID': loaders.IntFromBinaryString, 46 'ShipGroupID': loaders.IntFromBinaryString, 47 'certificateTemplateID': loaders.IntFromBinaryString, 48 'factionID': loaders.IntFromBinaryString} 49 50 def sizeof_fmt(num): 51 for x in [' bytes', 52 ' KB', 53 ' MB', 54 ' GB']: 55 if num < 1024.0 and num > -1024.0: 56 return '%3.1f%s' % (num, x) 57 num /= 1024.0 58 59 return '%3.1f%s' % (num, ' TB') 60 61 62 class LoaderState(object): 63 64 def __init__(self, factories, logger = None, cfgObject = None): 65 self.factories = factories 66 67 def RepresentSchemaNode(self, data, offset, path, schemaNode): 68 schemaType = schemaNode.get('type') 69 if schemaType in self.factories: 70 return self.factories[schemaType](data, offset, schemaNode, path, self) 71 raise NotImplementedError("Unknown type not supported in binary loader '%s'" % str(schemaType)) 72 73 def FormatSize(self, size): 74 return sizeof_fmt(size) 75 76 77 def RepresentSchemaNode(data, offset, schemaNode, path, extraState = None): 78 schemaType = schemaNode.get('type') 79 if extraState is None: 80 extraState = LoaderState(defaultLoaderFactories, None) 81 if schemaType in extraState.factories: 82 return extraState.factories[schemaType](data, offset, schemaNode, path, extraState) 83 raise NotImplementedError("Unknown type not supported in binary loader '%s'" % str(schemaType)) 84 85 86 def LoadFromString(dataString, optimizedSchema = None, path = None, extraState = None): 87 if path is None: 88 path = FsdDataPathObject('<string input>') 89 offsetToData = 0 90 if optimizedSchema is None: 91 schemaSize = uint32.unpack_from(dataString, 0)[0] 92 optimizedSchema = cPickle.loads(dataString[4:schemaSize + 4]) 93 offsetToData = schemaSize + 4 94 dataBuffer = ctypes.create_string_buffer(dataString, len(dataString)) 95 return RepresentSchemaNode(dataBuffer, offsetToData, optimizedSchema, path, extraState) 96 97 98 def GetEmbeddedSchemaAndSizeFromFile(fileObject): 99 schemaSize = uint32.unpack(fileObject.read(4))[0] 100 pickledSchema = fileObject.read(schemaSize) 101 return (cPickle.loads(pickledSchema), schemaSize) 102 103 104 def LoadIndexFromFile(fileObject, optimizedSchema = None, cacheItems = 100, path = None, extraState = None): 105 if extraState is None: 106 extraState = LoaderState(defaultLoaderFactories, None) 107 if path is None: 108 path = FsdDataPathObject('<file input>') 109 offsetToData = 0 110 if optimizedSchema is None: 111 fileObject.seek(0) 112 optimizedSchema, schemaSize = GetEmbeddedSchemaAndSizeFromFile(fileObject) 113 offsetToData = schemaSize + 4 114 if optimizedSchema.get('multiIndex', False): 115 return loaders.dictLoader.MultiIndexLoader(fileObject, cacheItems, optimizedSchema, path, extraState, offsetToData=offsetToData) 116 else: 117 return loaders.dictLoader.IndexLoader(fileObject, cacheItems, optimizedSchema, path, extraState, offsetToData=offsetToData) 118 119 120 class BlueResFileIOWrapper(object): 121 122 def __init__(self, resFile): 123 self.__resFile__ = resFile 124 125 def seek(self, offset, start = None): 126 if start is None: 127 return self.__resFile__.seek(offset) 128 else: 129 if start == os.SEEK_END: 130 offset = -offset 131 return self.__resFile__.seek(offset, start) 132 133 def read(self, bytes): 134 return self.__resFile__.Read(bytes) 135 136 def tell(self): 137 return self.__resFile__.pos 138 139 140 def LoadFSDDataForCFG(dataResPath, schemaResPath = None, optimize = True, cacheSize = 100): 141 import blue 142 from . import schemaOptimizer 143 res = blue.ResFile() 144 if schemaResPath is not None: 145 if not res.Open(schemaResPath): 146 log.error('Could not load FSD data schema: %s', schemaResPath) 147 else: 148 schema = persistence.LoadSchema(res.Read()) 149 if optimize: 150 schema = persistence.GetUnOptimizedRuntimeSchema(schema) 151 schema = schemaOptimizer.OptimizeSchema(schema, boot.role.capitalize()) 152 else: 153 schema = None 154 if not res.Open(dataResPath): 155 log.error('Could not load FSD data file: %s', dataResPath) 156 else: 157 wrappedRes = BlueResFileIOWrapper(res) 158 if schema is None: 159 peekSchema, size = GetEmbeddedSchemaAndSizeFromFile(wrappedRes) 160 wrappedRes.seek(0) 161 else: 162 peekSchema = schema 163 if peekSchema['type'] == 'dict' and peekSchema.get('buildIndex', False): 164 return LoadIndexFromFile(wrappedRes, schema, cacheSize, path=FsdDataPathObject('<file %s>' % dataResPath)) 165 s = res.Read() 166 log.info('Loading FSD data file %s into memory. %s', dataResPath, sizeof_fmt(len(s))) 167 return LoadFromString(s, schema, path=FsdDataPathObject('<file %s>' % dataResPath)) 168 169 170 def LoadFSDDataInPython(dataResPath, schemaResPath = None, optimize = None, cacheSize = 100): 171 from . import schemaOptimizer 172 if schemaResPath is not None: 173 with open(schemaResPath, 'r') as schemaFile: 174 schema = persistence.LoadSchema(schemaFile.read()) 175 if optimize: 176 schema = persistence.GetUnOptimizedRuntimeSchema(schema) 177 schema = schemaOptimizer.OptimizeSchema(schema, optimize) 178 else: 179 schema = None 180 dataFile = open(dataResPath, 'rb') 181 if schema is None: 182 peekSchema, size = GetEmbeddedSchemaAndSizeFromFile(dataFile) 183 dataFile.seek(0) 184 else: 185 peekSchema = schema 186 if peekSchema['type'] == 'dict' and peekSchema.get('buildIndex', False): 187 return LoadIndexFromFile(dataFile, schema, cacheSize, path=FsdDataPathObject('<file %s>' % dataResPath)) 188 else: 189 s = dataFile.read() 190 log.info('Loading FSD data file %s into memory. %s', dataResPath, sizeof_fmt(len(s))) 191 dataFile.close() 192 return LoadFromString(s, schema, path=FsdDataPathObject('<file %s>' % dataResPath)) 193 194 195 def LoadFSDDataForCFGOrFail(dataResPath, schemaResPath = None, optimize = True, cacheSize = 100): 196 data = LoadFSDDataForCFG(dataResPath, schemaResPath, optimize, cacheSize) 197 if data is None: 198 errorString = 'Could not load FSD data for {dataResPath}'.format(dataResPath=dataResPath) 199 raise RuntimeError(errorString) 200 return data