NovaBryan/EVE_Dump
Log | Files | 📦 下载最新源码 (.zip) | Refs | README

loaders.py (4409B)

      1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\fsdUtilities\loaders.py
      2 from __future__ import print_function
      3 import re
      4 import time
      5 import os
      6 import yaml
      7 from fsd.mapReduce import MapReduceMany, defaultWalker
      8 from fsd.common.pathSpecifiers import PathConditional, GetPathSpecifierRegex
      9 from fsd.mapReduce.operations import MapReduceFinalizeJob, MapOperation
     10 from fsd.schemas.binaryLoader import LoadFromString
     11 from fsd.schemas.binaryRepresenter import RepresentAsBinaryWithEmbeddedSchema
     12 from fsd.schemas.schemaOptimizer import OptimizeSchema
     13 from fsd.schemas.persistence import SafeSchemaLoader
     14 from fsdUtilities.pathUtils import GetStaticDataPath, GetAbsoluteBranchPath
     15 from fsd import AbsJoin
     16 from brennivin.yamlext import loadfile
     17 
     18 def LoadSchema(schemaPath):
     19     if '#' in schemaPath:
     20         filePath, schemaPath = schemaPath.split('#')
     21     else:
     22         filePath, schemaPath = schemaPath, ''
     23     if os.path.exists(GetStaticDataPath(filePath)):
     24         filePath = GetStaticDataPath(filePath)
     25     with open(filePath, 'r') as schemaFile:
     26         data = yaml.load(schemaFile, Loader=SafeSchemaLoader)
     27     for part in schemaPath.split('.'):
     28         if part:
     29             data = data[part]
     30 
     31     return data
     32 
     33 
     34 def LoadDataFromFile(filePath):
     35     if os.path.exists(GetStaticDataPath(filePath)):
     36         pathToFile = GetStaticDataPath(filePath)
     37     else:
     38         pathToFile = filePath
     39     return loadfile(pathToFile)
     40 
     41 
     42 def _MapData(rootPath, relativePath, filesInHeirachy, fileContent):
     43     yield (relativePath, fileContent)
     44 
     45 
     46 def _FinalizeData(rootPath, results):
     47     return {path:content for path, content in results}
     48 
     49 
     50 def LoadDataFromMultipleFiles(root, includePath, showLog = True):
     51     if os.path.exists(GetStaticDataPath(root)):
     52         pathToRoot = GetStaticDataPath(root)
     53     else:
     54         pathToRoot = root
     55     pathRegularExpression = GetPathSpecifierRegex(includePath)
     56     data = {}
     57     startTime = time.time()
     58     for root, dirs, files in os.walk(pathToRoot):
     59         for filePath in files:
     60             if re.match(pathRegularExpression, filePath):
     61                 dataKey = _RemoveSimilarEndSubstring(filePath, includePath)
     62                 try:
     63                     dataKey = int(dataKey)
     64                 except ValueError:
     65                     pass
     66 
     67                 data[dataKey] = loadfile(AbsJoin(root, filePath))
     68 
     69     if showLog:
     70         print('Parsed %d files in %.1f sec for %s' % (len(data), time.time() - startTime, GetAbsoluteBranchPath(root, includePath)))
     71     return data
     72 
     73 
     74 def LoadDataFromMultipleFilesWithMultiProcessing(root, includePath, mapFunction = _MapData, finalizeFunction = _FinalizeData):
     75     if os.path.exists(GetStaticDataPath(root)):
     76         pathToRoot = GetStaticDataPath(root)
     77     else:
     78         pathToRoot = root
     79     includeConditionals = PathConditional(includes=includePath)
     80     walker = defaultWalker.MapReduceDirectoryWalker(pathToRoot, loadfile, GetAbsoluteBranchPath('eve/autobuild/temp/staticData/utilities'))
     81     reduceFinalizeJob = MapReduceFinalizeJob('objectData', [MapOperation('MapData', mapFunction)], pathConditional=includeConditionals, finalizeFunction=finalizeFunction)
     82     import multiprocessing
     83     pool = multiprocessing.Pool(processes=4)
     84     startTime = time.time()
     85     generatedResults = MapReduceMany(walker, [], includePath, [], [reduceFinalizeJob], pool)
     86     generatedResults = generatedResults['objectData']
     87     filePath = GetAbsoluteBranchPath(root, includePath)
     88     print('Parsed %d files in %.1f sec for %s' % (len(generatedResults), time.time() - startTime, filePath))
     89     result = {}
     90     for key, value in generatedResults.iteritems():
     91         shortenedKey = _RemoveSimilarEndSubstring(key, includePath)
     92         try:
     93             shortenedKey = int(shortenedKey)
     94         except ValueError:
     95             pass
     96 
     97         result[shortenedKey] = value
     98 
     99     return result
    100 
    101 
    102 def _RemoveSimilarEndSubstring(key, path):
    103     reversedPath = path[::-1]
    104     reversedKey = key[::-1]
    105     index = 0
    106     while len(reversedKey) != 0 and index < len(reversedPath) and reversedKey[0] == reversedPath[index]:
    107         reversedKey = reversedKey[1:]
    108         index = index + 1
    109 
    110     return reversedKey[::-1]
    111 
    112 
    113 def CreateFsdObjectFromData(data, schema, target = 'client'):
    114     schema = LoadSchema(schema)
    115     binaryBlob = RepresentAsBinaryWithEmbeddedSchema(data, OptimizeSchema(schema, target))
    116     return LoadFromString(binaryBlob)