NovaBryan/EVE_Dump
Log | Files | 📦 下载最新源码 (.zip) | Refs | README

downloader.py (3827B)

      1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\imagecaching\client\repository\downloader.py
      2 import logging
      3 import blue
      4 import requests
      5 from imagecaching.client.compat import to_text
      6 from imagecaching.client.const import LOGGER_NAME, extension_for_content_type
      7 from six.moves.http_client import OK, NOT_MODIFIED
      8 logger = logging.getLogger(LOGGER_NAME)
      9 MAX_ATTEMPTS = 4
     10 REQUEST_TIMEOUT_SEC = 10
     11 HTTPS_PREFIX = u'https://'
     12 
     13 def _default_sleep(milliseconds):
     14     blue.pyos.synchro.SleepWallclock(milliseconds)
     15 
     16 
     17 class DownloadResult(object):
     18     __slots__ = ['status',
     19      'data',
     20      'e_tag',
     21      'status_code',
     22      'error',
     23      'extension']
     24 
     25     def __init__(self, status, data = None, e_tag = None, status_code = None, error = None, extension = None):
     26         self.status = status
     27         self.data = data
     28         self.e_tag = e_tag
     29         self.status_code = status_code
     30         self.error = error
     31         self.extension = extension
     32 
     33 
     34 class ImageDownloader(object):
     35 
     36     def __init__(self, session = None, sleep = None):
     37         self._session = session or requests.session()
     38         self._sleep = sleep or _default_sleep
     39 
     40     def fetch(self, url, e_tag = None):
     41         url = url.strip()
     42         if not url.lower().startswith(HTTPS_PREFIX):
     43             logger.warn('Refusing to fetch a non-https url: %s', url)
     44             return DownloadResult(status=None, error=u'url is not https')
     45         request_url = url.encode('ascii')
     46         headers = {'If-None-Match': e_tag} if e_tag else {}
     47         response, exc = self._make_request(request_url, headers)
     48         if response is None:
     49             logger.warn('Retries exhausted for %s: %s', url, exc)
     50             return DownloadResult(status=None, error=u'%s' % (exc,) if exc else u'request failed')
     51         final_url = to_text(response.url) or u''
     52         if not final_url.lower().startswith(HTTPS_PREFIX):
     53             logger.warn('Refusing a redirect off https for %s: ended at %s', url, final_url)
     54             return DownloadResult(status=None, status_code=response.status_code, error=u'redirected to a non-https url')
     55         if response.status_code == NOT_MODIFIED:
     56             return DownloadResult(status=NOT_MODIFIED, status_code=response.status_code)
     57         if response.status_code != OK:
     58             logger.warn('Unexpected response for %s: status=%s', url, response.status_code)
     59             return DownloadResult(status=None, status_code=response.status_code, error=u'unexpected response (status=%s)' % response.status_code)
     60         content_type = response.headers.get('Content-Type', None)
     61         extension = extension_for_content_type(content_type)
     62         if extension is None:
     63             logger.warn('Refusing an unsupported content type for %s: %s', url, content_type)
     64             return DownloadResult(status=None, status_code=response.status_code, error=u'unsupported image content-type (%s)' % content_type)
     65         return DownloadResult(status=OK, data=response.content, e_tag=response.headers.get('ETag'), status_code=response.status_code, extension=extension)
     66 
     67     def _make_request(self, request_url, headers):
     68         response = None
     69         exc = None
     70         for attempt in range(MAX_ATTEMPTS):
     71             try:
     72                 response = self._session.get(request_url, headers=headers, timeout=REQUEST_TIMEOUT_SEC, allow_redirects=True)
     73                 break
     74             except Exception as e:
     75                 exc = e
     76                 if attempt + 1 < MAX_ATTEMPTS:
     77                     backoff_seconds = 2 ** attempt
     78                     logger.warn('Retrying requesting image for url: %s. Sleeping for %s seconds. Attempts done so far: (%s/%s). Exception: %s', request_url, backoff_seconds, attempt + 1, MAX_ATTEMPTS, e)
     79                     self._sleep(backoff_seconds * 1000)
     80 
     81         return (response, exc)