repository.py (6413B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\imagecaching\client\repository\repository.py 2 import logging 3 import os 4 import re 5 import sys 6 import blue 7 from imagecaching.client.api.error import ImageCacheError 8 from imagecaching.client.api.signal import on_image_cached, on_image_deleted 9 from imagecaching.client.const import DEFAULT_BASE_PATH, IMAGE_EXTENSIONS, LOGGER_NAME, OUTCOME_NOT_MODIFIED, OUTCOME_FAILED, OUTCOME_DOWNLOADED, OUTCOME_SERVED_STALE 10 from imagecaching.client.model.definitions.image.attributes import ImageAttributes 11 from imagecaching.client.model.definitions.image.entry import ImageEntry 12 from imagecaching.client.model.definitions.image.identifier import ImageIdentifier 13 from imagecaching.client.repository.downloader import ImageDownloader 14 from imagecaching.client.repository.flatbuffer_index import FolderIndex 15 from locks.singleFlight import single_flight 16 from six.moves.http_client import NOT_MODIFIED 17 logger = logging.getLogger(LOGGER_NAME) 18 _MOUNT_PREFIX = re.compile('^[A-Za-z0-9_]+:/') 19 20 def _resolve_path_for_writing(res_path): 21 return blue.paths.ResolvePathForWriting(res_path) 22 23 24 class ImageCacheRepository(object): 25 26 def __init__(self, downloader = None): 27 self._downloader = downloader or ImageDownloader() 28 self.last_status_code = None 29 self.last_error = None 30 self.last_outcome = None 31 32 @single_flight 33 def cache_image(self, image_id, url, cache_path): 34 cache_path = self._resolve_uri(cache_path) 35 folder = cache_path.rsplit(u'/', 1)[0] 36 index = FolderIndex(folder) 37 index.load() 38 identifier = ImageIdentifier(image_id) 39 existing = index.get(identifier) 40 cached = existing if existing and self._image_file_exists(existing.attributes.res_path) else None 41 result = self._downloader.fetch(url, cached.attributes.e_tag if cached else None) 42 self.last_status_code = result.status_code 43 self.last_error = result.error 44 if result.status is None: 45 self.last_outcome = OUTCOME_SERVED_STALE if cached else OUTCOME_FAILED 46 if cached: 47 return cached.attributes.res_path 48 return 49 if result.status == NOT_MODIFIED: 50 if cached: 51 self.last_outcome = OUTCOME_NOT_MODIFIED 52 return cached.attributes.res_path 53 self.last_outcome = OUTCOME_FAILED 54 return 55 base, old_ext = os.path.splitext(cache_path) 56 if old_ext.lower() in IMAGE_EXTENSIONS: 57 cache_path = base + result.extension 58 else: 59 cache_path += result.extension 60 self._write_image_file(cache_path, result.data) 61 self._invalidate_res_manager(cache_path) 62 if existing and existing.attributes.res_path != cache_path: 63 self._delete_image_file(existing.attributes.res_path) 64 self._invalidate_res_manager(existing.attributes.res_path) 65 entry = ImageEntry(identifier, ImageAttributes(cache_path, result.e_tag)) 66 index.update(lambda idx: idx.upsert(entry)) 67 self.last_outcome = OUTCOME_DOWNLOADED 68 on_image_cached(image_id, cache_path) 69 return cache_path 70 71 def delete_image(self, image_id, cache_path): 72 cache_path = self._resolve_uri(cache_path) 73 folder = cache_path.rsplit(u'/', 1)[0] 74 index = FolderIndex(folder) 75 index.load() 76 identifier = ImageIdentifier(image_id) 77 existing = index.get(identifier) 78 res_path = existing.attributes.res_path if existing else cache_path 79 self._delete_image_file(res_path) 80 self._invalidate_res_manager(res_path) 81 removed = [False] 82 83 def _remove(idx): 84 removed[0] = idx.remove(identifier) 85 86 index.update(_remove) 87 if removed[0]: 88 on_image_deleted(image_id, res_path) 89 return removed[0] 90 91 def list_entries(self, cache_folder): 92 index = self._load_folder_index(cache_folder) 93 return [ index.get(identifier) for identifier in index.identifiers() ] 94 95 def list_image_ids(self, cache_folder): 96 return [ entry.identifier.image_id for entry in self.list_entries(cache_folder) ] 97 98 def prune_folder(self, cache_folder, keep_image_ids): 99 keep = set(keep_image_ids) 100 deleted = [] 101 for entry in self.list_entries(cache_folder): 102 image_id = entry.identifier.image_id 103 if image_id in keep: 104 continue 105 if self.delete_image(image_id, entry.attributes.res_path): 106 deleted.append(image_id) 107 108 return deleted 109 110 @classmethod 111 def _load_folder_index(cls, cache_folder): 112 index = FolderIndex(cls._resolve_uri(cache_folder)) 113 index.load() 114 return index 115 116 @staticmethod 117 def _resolve_uri(cache_path): 118 if _MOUNT_PREFIX.match(cache_path): 119 return cache_path 120 return u'%s/%s' % (DEFAULT_BASE_PATH, cache_path.lstrip(u'/')) 121 122 @staticmethod 123 def _write_image_file(cache_path, data): 124 folder_path = _resolve_path_for_writing(cache_path.rsplit(u'/', 1)[0]) 125 try: 126 os.makedirs(folder_path) 127 except OSError: 128 if not os.path.isdir(folder_path): 129 raise 130 logger.warning('Failed to create folder %r', folder_path) 131 132 resfile = blue.classes.CreateInstance('blue.ResFile') 133 opened = resfile.Open(cache_path, 0) 134 if not opened: 135 opened = resfile.Create(cache_path) 136 if not opened: 137 raise ImageCacheError(u'Could not open cache file for writing: %s' % cache_path) 138 try: 139 resfile.Write(data) 140 finally: 141 resfile.Close() 142 143 @staticmethod 144 def _image_file_exists(cache_path): 145 return os.path.exists(_resolve_path_for_writing(cache_path)) 146 147 @staticmethod 148 def _delete_image_file(cache_path): 149 try: 150 os.remove(_resolve_path_for_writing(cache_path)) 151 except OSError: 152 logger.warning(u'Could not remove cache file for writing: %s' % cache_path) 153 154 @staticmethod 155 def _invalidate_res_manager(cache_path): 156 try: 157 res = blue.resMan.GetResource(str(cache_path), 'atlas') 158 if res: 159 res.Reload() 160 except SystemError: 161 logger.warning(u'Could not invalidate cache file for writing: %s' % cache_path)