storage.py (17892B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\fsdlite\storage.py 2 from __future__ import print_function 3 import os 4 import time 5 import fnmatch 6 import fsdlite 7 import threading 8 import weakref 9 import collections 10 import logging 11 from fsd import GetBranchRoot, AbsJoin, GetFullName 12 try: 13 from watchdog.events import FileSystemEventHandler 14 except ImportError: 15 FileSystemEventHandler = object 16 17 try: 18 from scandir import walk 19 except ImportError: 20 from os import walk 21 22 from eveprefs import boot 23 logger = logging.getLogger(__name__) 24 try: 25 import P4 26 except ImportError: 27 P4 = None 28 29 try: 30 from blinker import signal 31 except ImportError: 32 signal = None 33 34 FORCE_FILE_MONITOR = None 35 36 class Storage(collections.MutableMapping): 37 38 def __init__(self, data = None, cache = None, mapping = None, indexes = None, monitor = False, coerce = None, readonly = False): 39 self.extension = '.staticdata' 40 self.cache_path = cache 41 self.cache_filename = self._get_cache_filename(self.cache_path) 42 self.subscribe_close_cache_signal() 43 self.coerce = coerce or str 44 self.immutable = True 45 self.file_monitor = None 46 self._files = None 47 self._fileLoaderThread = None 48 self.objects = {} 49 self._path = GetFullName(os.path.abspath(data)) if data else None 50 self._cache = None 51 self._mapping = mapping 52 self._indexes = indexes 53 if FORCE_FILE_MONITOR is None: 54 self.monitor = monitor 55 else: 56 self.monitor = FORCE_FILE_MONITOR 57 self.waiting = False 58 self.readonly = readonly 59 60 def initializeFileIndex(self): 61 if self._fileLoaderThread is None: 62 self._fileLoaderThread = FileLoaderThread(self) 63 self._fileLoaderThread.start() 64 65 def invalidateFileIndex(self): 66 if self._fileLoaderThread is not None: 67 self._fileLoaderThread.kill() 68 self._files = None 69 self._fileLoaderThread = None 70 71 def resetFileIndex(self): 72 self.invalidateFileIndex() 73 self.initializeFileIndex() 74 75 @property 76 def files(self): 77 self.initializeFileIndex() 78 if self._fileLoaderThread.isAlive(): 79 self._fileLoaderThread.join() 80 return self._files 81 82 def __getitem__(self, key): 83 if key is None: 84 raise KeyError 85 try: 86 return self._object_load(key) 87 except KeyError: 88 pass 89 90 try: 91 file_time = self._file_time(key) 92 cache_time = self._cache_time(key) 93 if cache_time >= file_time: 94 return self._cache_load(key) 95 except KeyError: 96 pass 97 98 return self._file_load(key) 99 100 def __setitem__(self, key, item): 101 if self.immutable: 102 raise RuntimeError('FSD Storage is Immutable') 103 old_monitor, self.monitor = self.monitor, False 104 self._file_save(key, item) 105 self._cache_save(key, item) 106 self._object_save(key, item) 107 self.monitor = old_monitor 108 109 def __delitem__(self, key): 110 if self.immutable: 111 raise RuntimeError('FSD Storage is Immutable') 112 if self.path: 113 try: 114 filepath = self._file_path(key) 115 Delete(filepath) 116 self._file_index(filepath) 117 return 118 except P4.P4Exception as exception: 119 raise KeyError(exception) 120 121 def __len__(self): 122 return len(self.keys()) 123 124 def __iter__(self): 125 return iter(self.keys()) 126 127 def __enter__(self): 128 if not self.readonly: 129 self.immutable = False 130 131 def __exit__(self, *args): 132 self.immutable = True 133 134 def __contains__(self, key): 135 try: 136 key = self.coerce(key) 137 except TypeError: 138 return False 139 140 if key in self.objects: 141 return True 142 elif self.files: 143 return key in self.files 144 elif self.cache: 145 return key in self.cache 146 else: 147 return False 148 149 @property 150 def path(self): 151 return self._path 152 153 @property 154 def mapping(self): 155 return self._mapping 156 157 @property 158 def indexes(self): 159 return self._indexes 160 161 @path.setter 162 def path(self, value): 163 if self.objects: 164 self.objects.clear() 165 if self.cache: 166 self.cache.clear() 167 self._path = GetFullName(os.path.abspath(value)) 168 self.resetFileIndex() 169 self.monitor = self.monitor 170 171 def keys(self): 172 if self.files: 173 return [ self.coerce(key) for key in self.files.keys() ] 174 if self.cache: 175 return [ self.coerce(key) for key in self.cache.keys() ] 176 return [] 177 178 def Get(self, key): 179 return self[key] 180 181 def _get_cache_filename(self, cache_path): 182 if cache_path: 183 _, filename = os.path.split(cache_path) 184 return filename 185 186 def prime(self, path = None): 187 keys = set(self.keys()) 188 for key, (filename, timestamp) in self.files.iteritems(): 189 if path is None or fnmatch.fnmatch(filename, path): 190 keys.add(self.coerce(key)) 191 192 for key in keys: 193 self[key] 194 195 def prime_key(self, key): 196 self[key] 197 198 def prime_file(self, file_path): 199 self._file_index(file_path) 200 201 def filter_keys(self, name, key): 202 if self.cache: 203 return [ self.coerce(key) for key in self.cache.index('{}.{}'.format(name, key)) ] 204 return [] 205 206 def filter(self, name, key): 207 return [ self[key] for key in self.filter_keys(name, key) ] 208 209 def index(self, name, key): 210 try: 211 return self.filter(name, key)[0] 212 except IndexError: 213 raise KeyError 214 215 def key_path(self, key): 216 return self._file_path(key) 217 218 def signal_receiver(self, filepaths): 219 if self.cache_path in filepaths: 220 self.close_cache_connection() 221 222 def subscribe_close_cache_signal(self): 223 if signal is not None: 224 sub = signal('building_static_datafiles') 225 sub.connect(self.signal_receiver) 226 227 def disable_cache(self): 228 if self._cache: 229 self._cache.close() 230 self._cache = None 231 if self.cache_path: 232 self.cache_path = None 233 234 @classmethod 235 def close_cache_connections(cls): 236 for _, connection in fsdlite.Cache._connections.iteritems(): 237 connection.commit() 238 connection.close() 239 240 fsdlite.Cache._connections = {} 241 242 def close_cache_connection(self): 243 if self._cache is not None: 244 connection = self._cache._connections.get(self.cache_path, None) 245 if connection: 246 connection.commit() 247 connection.close() 248 if self.cache_path in fsdlite.Cache._connections: 249 del fsdlite.Cache._connections[self.cache_path] 250 251 def _object_load(self, key): 252 return self.objects[self.coerce(key)] 253 254 def _object_save(self, key, obj): 255 key = self.coerce(key) 256 self.objects[key] = obj 257 258 def _object_discard(self, key): 259 key = self.coerce(key) 260 self.objects.pop(key, None) 261 262 def _cache_load(self, key): 263 if self.cache: 264 data = self.cache[self.coerce(key)] 265 obj = fsdlite.decode(data, mapping=self.mapping, json=True) 266 self._object_save(key, obj) 267 return obj 268 raise KeyError('No Cache') 269 270 def _cache_save(self, key, obj): 271 if self.cache: 272 key = self.coerce(key) 273 self.cache[key] = fsdlite.encode(obj, json=True) 274 self.cache.index_clear(key) 275 for indexName, indexKeys in fsdlite.index(obj, self.indexes).iteritems(): 276 for indexKey in indexKeys: 277 self.cache.index_set('{}.{}'.format(indexName, indexKey), key) 278 279 def _cache_discard(self, key): 280 if self.cache: 281 try: 282 key = self.coerce(key) 283 del self.cache[key] 284 self.cache.index_clear(key) 285 except KeyError: 286 pass 287 288 def _cache_time(self, key): 289 if self.cache: 290 try: 291 return self.cache.time(self.coerce(key)) 292 except KeyError: 293 pass 294 295 return 0 296 297 def _set_cache(self, value): 298 if value is not None and not isinstance(value, str): 299 raise ValueError('Cache path must be a string') 300 self.cache_path = value 301 self._cache = None 302 303 def _get_cache(self): 304 if self.cache_path and self._cache is None: 305 self._cache = fsdlite.Cache(self.cache_path) 306 return self._cache 307 308 cache = property(_get_cache, _set_cache) 309 310 def file_init(self): 311 if self._files is None: 312 logger.debug("Initializing file cache for path '{}'".format(self.path)) 313 self._files = {} 314 if self.path and os.path.isdir(self.path): 315 for base, directories, files in walk(self.path): 316 for filename in files: 317 self._file_index(os.path.join(base, filename)) 318 319 logger.debug('File cache initialization complete') 320 321 def _file_index(self, filename): 322 if fnmatch.fnmatch(filename, '*' + self.extension): 323 key = self.coerce(self._file_key(filename)) 324 if os.path.exists(filename): 325 modified = os.path.getmtime(filename) 326 self._files[key] = (filename, modified) 327 else: 328 self._files.pop(key, None) 329 return key 330 331 def _file_changed(self, event, filename): 332 key = self._file_index(filename) 333 if key is not None: 334 self._object_discard(key) 335 self._cache_discard(key) 336 self.prime_key(key) 337 self._on_file_load(key) 338 self.waiting = False 339 340 def _file_deleted(self, event, filepath): 341 if not os.path.exists(filepath): 342 key = self._file_index(filepath) 343 if key is not None: 344 self._object_discard(key) 345 self._cache_discard(key) 346 347 def _on_file_load(self, key): 348 pass 349 350 def _file_index_on_key(self, key): 351 file_path = self._file_path(key) 352 if not os.path.exists(file_path): 353 return 354 self._file_index(file_path) 355 356 def _file_load(self, key): 357 if self.path: 358 try: 359 key = self.coerce(key) 360 if key not in self.files: 361 self._file_index_on_key(key) 362 filepath, modified = self.files[key] 363 with open(filepath, 'r') as stream: 364 data = fsdlite.load(stream.read()) 365 self._cache_save(key, data) 366 obj = fsdlite.decode(data, mapping=self.mapping) 367 self._object_save(key, obj) 368 return obj 369 except IOError as exception: 370 raise KeyError(exception) 371 372 raise KeyError('No Static Data') 373 374 def _file_save(self, key, obj): 375 if self.path: 376 try: 377 filepath = self._file_path(key) 378 Checkout(filepath) 379 with open(filepath, 'w') as stream: 380 stream.write(fsdlite.dump(fsdlite.strip(obj))) 381 self._file_index(filepath) 382 return 383 except IOError as exception: 384 raise KeyError(exception) 385 386 raise KeyError('No Static Data') 387 388 def _file_time(self, key): 389 if self.path: 390 try: 391 return self.files[self.coerce(key)][1] 392 except KeyError: 393 return time.time() 394 395 return 0 396 397 def _file_path(self, key): 398 if self.path: 399 return GetFullName(AbsJoin(self.path, str(key) + self.extension)) 400 raise RuntimeError('Unable to determine file path, no data path provided') 401 402 def _file_key(self, path): 403 if self.path: 404 key, _ = os.path.splitext(os.path.relpath(path, self.path)) 405 return key.replace('\\', '/') 406 raise RuntimeError('Unable to determine file key, no data path provided') 407 408 def _get_monitor(self): 409 return self.file_monitor is not None 410 411 def _set_monitor(self, value): 412 monitor = getattr(self, 'file_monitor', None) 413 if monitor: 414 fsdlite.stop_file_monitor(monitor) 415 self.file_monitor = None 416 if value and self.path: 417 file_handler = FileHandler(fsdlite.WeakMethod(self._file_changed), fsdlite.WeakMethod(self._file_deleted)) 418 self.file_monitor = fsdlite.start_file_monitor(self.path, file_handler=file_handler) 419 420 monitor = property(_get_monitor, _set_monitor) 421 422 def wait(self): 423 self.waiting = True 424 while self.waiting: 425 pass 426 427 428 class FileHandler(FileSystemEventHandler): 429 430 def __init__(self, modified_callback, deleted_callback): 431 self.modified_callback = modified_callback 432 self.deleted_callback = deleted_callback 433 434 def on_created(self, event): 435 if not event.is_directory: 436 try: 437 self.modified_callback(event.event_type, event.src_path) 438 except Exception: 439 logging.exception('Error handling file monitor on created event') 440 441 def on_modified(self, event): 442 if not event.is_directory and os.path.exists(event.src_path): 443 try: 444 self.modified_callback(event.event_type, event.src_path) 445 except Exception: 446 logging.exception('Error handling file monitor on modified event') 447 448 def on_moved(self, event): 449 if not event.is_directory: 450 try: 451 self.modified_callback(event.event_type, event.src_path) 452 except Exception: 453 logging.exception('Error handling file monitor on moved event') 454 455 def on_deleted(self, event): 456 if not event.is_directory: 457 try: 458 self.deleted_callback(event.event_type, event.src_path) 459 except Exception: 460 logging.exception('Error handling file monitor on deleted event') 461 462 463 class WeakStorage(Storage): 464 465 def __init__(self, *args, **kwargs): 466 Storage.__init__(self, *args, **kwargs) 467 self.objects = weakref.WeakValueDictionary() 468 469 470 class EveStorage(Storage): 471 472 def __init__(self, data, cache, on_file_load = None, *args, **kwargs): 473 monitor = kwargs.get('monitor', True) 474 try: 475 import blue 476 if blue.pyos.packaged: 477 monitor = False 478 data = None 479 if boot.role == 'client': 480 import remotefilecache 481 logger.info('Prefetching %s', cache) 482 cache = remotefilecache.prefetch_single_file('res:/staticdata/' + cache, verify=True) 483 kwargs['readonly'] = True 484 logger.info('Prefetched %s', cache) 485 else: 486 cache = os.path.join(blue.paths.ResolvePath(u'res:/staticdata'), cache) 487 elif ShouldUseSource(): 488 data = os.path.join(blue.paths.ResolvePath(u'root:/staticData'), data) 489 cache = os.path.join(blue.paths.ResolvePath(u'root:/autobuild/staticData/server'), cache) 490 elif os.path.exists(blue.paths.ResolvePath(u'res:/staticdata')): 491 data = None 492 cache = os.path.join(blue.paths.ResolvePath(u'res:/staticdata'), cache) 493 else: 494 raise ImportError 495 except ImportError: 496 if data is not None and ShouldUseSource(): 497 data = AbsJoin(GetBranchRoot(), 'eve/staticData/', data) 498 if not os.path.exists(data): 499 data = None 500 if cache is not None: 501 cache = AbsJoin(GetBranchRoot(), 'eve/autobuild/staticData/server', cache) 502 if not os.path.exists(os.path.dirname(cache)): 503 cache = None 504 if on_file_load is not None: 505 self._on_file_load = on_file_load 506 507 kwargs['monitor'] = monitor 508 Storage.__init__(self, data, cache, *args, **kwargs) 509 510 511 class FileLoaderThread(threading.Thread): 512 513 def __init__(self, storage): 514 super(FileLoaderThread, self).__init__() 515 self._storage = storage 516 517 def run(self): 518 self._storage.file_init() 519 520 521 SHOULD_USE_SOURCE = None 522 523 def ShouldUseSource(): 524 global SHOULD_USE_SOURCE 525 if SHOULD_USE_SOURCE is None: 526 SHOULD_USE_SOURCE = bool(int(os.environ.get('EVE_FSDLITE_USE_SOURCE', '1'))) 527 if not SHOULD_USE_SOURCE: 528 print('FsdLite Storage will not use source files for this run as environment variable is set to False') 529 try: 530 import blue 531 except ImportError: 532 return True 533 534 return SHOULD_USE_SOURCE and os.path.exists(blue.paths.ResolvePath(u'root:/staticData')) 535 536 537 def DontUseSource(): 538 global SHOULD_USE_SOURCE 539 SHOULD_USE_SOURCE = False 540 541 542 def UseSource(): 543 global SHOULD_USE_SOURCE 544 SHOULD_USE_SOURCE = True 545 546 547 def Connection(): 548 if not hasattr(Connection, 'connection'): 549 connection = P4.P4() 550 connection.connect() 551 Connection.connection = connection 552 return Connection.connection 553 554 555 def Checkout(path): 556 if P4: 557 try: 558 connection = Connection() 559 try: 560 connection.run_edit(path) 561 except P4.P4Exception: 562 connection.run_add(path) 563 564 except P4.P4Exception as e: 565 logger.exception('Perforce error.. %s', e) 566 567 568 def Delete(path): 569 if P4: 570 try: 571 connection = Connection() 572 connection.run_delete(path) 573 except P4.P4Exception as e: 574 logger.exception('Perforce error.. %s', e)