cache.py (9280B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\lib\werkzeug\contrib\cache.py 2 import os 3 import re 4 try: 5 from hashlib import md5 6 except ImportError: 7 from md5 import new as md5 8 9 from itertools import izip 10 from time import time 11 from cPickle import loads, dumps, load, dump, HIGHEST_PROTOCOL 12 13 class BaseCache(object): 14 15 def __init__(self, default_timeout = 300): 16 self.default_timeout = default_timeout 17 18 def get(self, key): 19 return None 20 21 def delete(self, key): 22 pass 23 24 def get_many(self, *keys): 25 return map(self.get, keys) 26 27 def get_dict(self, *keys): 28 return dict(izip(keys, self.get_many(*keys))) 29 30 def set(self, key, value, timeout = None): 31 pass 32 33 def add(self, key, value, timeout = None): 34 pass 35 36 def set_many(self, mapping, timeout = None): 37 for key, value in mapping.iteritems(): 38 self.set(key, value, timeout) 39 40 def delete_many(self, *keys): 41 for key in keys: 42 self.delete(key) 43 44 def clear(self): 45 pass 46 47 def inc(self, key, delta = 1): 48 self.set(key, (self.get(key) or 0) + delta) 49 50 def dec(self, key, delta = 1): 51 self.set(key, (self.get(key) or 0) - delta) 52 53 54 class NullCache(BaseCache): 55 pass 56 57 58 class SimpleCache(BaseCache): 59 60 def __init__(self, threshold = 500, default_timeout = 300): 61 BaseCache.__init__(self, default_timeout) 62 self._cache = {} 63 self.clear = self._cache.clear 64 self._threshold = threshold 65 66 def _prune(self): 67 if len(self._cache) > self._threshold: 68 now = time() 69 for idx, (key, (expires, _)) in enumerate(self._cache.items()): 70 if expires <= now or idx % 3 == 0: 71 self._cache.pop(key, None) 72 73 def get(self, key): 74 now = time() 75 expires, value = self._cache.get(key, (0, None)) 76 if expires > time(): 77 return loads(value) 78 79 def set(self, key, value, timeout = None): 80 if timeout is None: 81 timeout = self.default_timeout 82 self._prune() 83 self._cache[key] = (time() + timeout, dumps(value, HIGHEST_PROTOCOL)) 84 85 def add(self, key, value, timeout = None): 86 if timeout is None: 87 timeout = self.default_timeout 88 if len(self._cache) > self._threshold: 89 self._prune() 90 item = (time() + timeout, dumps(value, HIGHEST_PROTOCOL)) 91 self._cache.setdefault(key, item) 92 93 def delete(self, key): 94 self._cache.pop(key, None) 95 96 97 _test_memcached_key = re.compile('[^\\x00-\\x21\\xff]{1,250}$').match 98 99 class MemcachedCache(BaseCache): 100 101 def __init__(self, servers, default_timeout = 300, key_prefix = None): 102 BaseCache.__init__(self, default_timeout) 103 if isinstance(servers, (list, tuple)): 104 try: 105 import cmemcache as memcache 106 is_cmemcache = True 107 except ImportError: 108 try: 109 import memcache 110 is_cmemcache = False 111 except ImportError: 112 raise RuntimeError('no memcache module found') 113 114 if is_cmemcache: 115 client = memcache.Client(map(str, servers)) 116 try: 117 client.debuglog = lambda *a: None 118 except: 119 pass 120 121 else: 122 client = memcache.Client(servers, False, HIGHEST_PROTOCOL) 123 else: 124 client = servers 125 self._client = client 126 self.key_prefix = key_prefix 127 128 def get(self, key): 129 if isinstance(key, unicode): 130 key = key.encode('utf-8') 131 if self.key_prefix: 132 key = self.key_prefix + key 133 if _test_memcached_key(key): 134 return self._client.get(key) 135 136 def get_dict(self, *keys): 137 key_mapping = {} 138 have_encoded_keys = False 139 for idx, key in enumerate(keys): 140 if isinstance(key, unicode): 141 encoded_key = key.encode('utf-8') 142 have_encoded_keys = True 143 else: 144 encoded_key = key 145 if self.key_prefix: 146 encoded_key = self.key_prefix + encoded_key 147 if _test_memcached_key(key): 148 key_mapping[encoded_key] = key 149 150 d = rv = self._client.get_multi(key_mapping.keys()) 151 if have_encoded_keys or self.key_prefix: 152 rv = {} 153 for key, value in d.iteritems(): 154 rv[key_mapping[key]] = value 155 156 if len(rv) < len(keys): 157 for key in keys: 158 if key not in rv: 159 rv[key] = None 160 161 return rv 162 163 def add(self, key, value, timeout = None): 164 if timeout is None: 165 timeout = self.default_timeout 166 if isinstance(key, unicode): 167 key = key.encode('utf-8') 168 if self.key_prefix: 169 key = self.key_prefix + key 170 self._client.add(key, value, timeout) 171 172 def set(self, key, value, timeout = None): 173 if timeout is None: 174 timeout = self.default_timeout 175 if isinstance(key, unicode): 176 key = key.encode('utf-8') 177 if self.key_prefix: 178 key = self.key_prefix + key 179 self._client.set(key, value, timeout) 180 181 def get_many(self, *keys): 182 d = self.get_dict(*keys) 183 return [ d[key] for key in keys ] 184 185 def set_many(self, mapping, timeout = None): 186 if timeout is None: 187 timeout = self.default_timeout 188 new_mapping = {} 189 for key, value in mapping.iteritems(): 190 if isinstance(key, unicode): 191 key = key.encode('utf-8') 192 if self.key_prefix: 193 key = self.key_prefix + key 194 new_mapping[key] = value 195 196 self._client.set_multi(new_mapping, timeout) 197 198 def delete(self, key): 199 if isinstance(key, unicode): 200 key = key.encode('utf-8') 201 if self.key_prefix: 202 key = self.key_prefix + key 203 if _test_memcached_key(key): 204 self._client.delete(key) 205 206 def delete_many(self, *keys): 207 new_keys = [] 208 for key in keys: 209 if isinstance(key, unicode): 210 key = key.encode('utf-8') 211 if self.key_prefix: 212 key = self.key_prefix + key 213 if _test_memcached_key(key): 214 new_keys.append(key) 215 216 self._client.delete_multi(new_keys) 217 218 def clear(self): 219 self._client.flush_all() 220 221 def inc(self, key, delta = 1): 222 if isinstance(key, unicode): 223 key = key.encode('utf-8') 224 if self.key_prefix: 225 key = self.key_prefix + key 226 self._client.incr(key, delta) 227 228 def dec(self, key, delta = 1): 229 if isinstance(key, unicode): 230 key = key.encode('utf-8') 231 if self.key_prefix: 232 key = self.key_prefix + key 233 self._client.decr(key, delta) 234 235 236 class GAEMemcachedCache(MemcachedCache): 237 238 def __init__(self, default_timeout = 300, key_prefix = None): 239 from google.appengine.api import memcache 240 MemcachedCache.__init__(self, memcache.Client(), default_timeout, key_prefix) 241 242 243 class FileSystemCache(BaseCache): 244 245 def __init__(self, cache_dir, threshold = 500, default_timeout = 300): 246 BaseCache.__init__(self, default_timeout) 247 self._path = cache_dir 248 self._threshold = threshold 249 if not os.path.exists(self._path): 250 os.makedirs(self._path) 251 252 def _prune(self): 253 entries = os.listdir(self._path) 254 if len(entries) > self._threshold: 255 now = time() 256 for idx, key in enumerate(entries): 257 try: 258 f = file(self._get_filename(key)) 259 if load(f) > now and idx % 3 != 0: 260 f.close() 261 continue 262 except: 263 f.close() 264 265 self.delete(key) 266 267 def _get_filename(self, key): 268 hash = md5(key).hexdigest() 269 return os.path.join(self._path, hash) 270 271 def get(self, key): 272 filename = self._get_filename(key) 273 try: 274 f = file(filename, 'rb') 275 try: 276 if load(f) >= time(): 277 return load(f) 278 finally: 279 f.close() 280 281 os.remove(filename) 282 except: 283 return 284 285 def add(self, key, value, timeout = None): 286 filename = self._get_filename(key) 287 if not os.path.exists(filename): 288 self.set(key, value, timeout) 289 290 def set(self, key, value, timeout = None): 291 if timeout is None: 292 timeout = self.default_timeout 293 filename = self._get_filename(key) 294 self._prune() 295 try: 296 f = file(filename, 'wb') 297 try: 298 dump(int(time() + timeout), f, 1) 299 dump(value, f, HIGHEST_PROTOCOL) 300 finally: 301 f.close() 302 303 except (IOError, OSError): 304 pass 305 306 def delete(self, key): 307 try: 308 os.remove(self._get_filename(key)) 309 except (IOError, OSError): 310 pass 311 312 def clear(self): 313 for key in os.listdir(self._path): 314 self.delete(key)