cookielib.py (41348B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\stdlib\cookielib.py 2 __all__ = ['Cookie', 3 'CookieJar', 4 'CookiePolicy', 5 'DefaultCookiePolicy', 6 'FileCookieJar', 7 'LWPCookieJar', 8 'lwp_cookie_str', 9 'LoadError', 10 'MozillaCookieJar'] 11 import re, urlparse, copy, time, urllib 12 try: 13 import threading as _threading 14 except ImportError: 15 import dummy_threading as _threading 16 17 import httplib 18 from calendar import timegm 19 debug = False 20 logger = None 21 22 def _debug(*args): 23 global logger 24 if not debug: 25 return 26 if not logger: 27 import logging 28 logger = logging.getLogger('cookielib') 29 return logger.debug(*args) 30 31 32 DEFAULT_HTTP_PORT = str(httplib.HTTP_PORT) 33 MISSING_FILENAME_TEXT = 'a filename was not supplied (nor was the CookieJar instance initialised with one)' 34 35 def _warn_unhandled_exception(): 36 import warnings, traceback, StringIO 37 f = StringIO.StringIO() 38 traceback.print_exc(None, f) 39 msg = f.getvalue() 40 warnings.warn('cookielib bug!\n%s' % msg, stacklevel=2) 41 42 43 EPOCH_YEAR = 1970 44 45 def _timegm(tt): 46 year, month, mday, hour, min, sec = tt[:6] 47 if year >= EPOCH_YEAR and 1 <= month <= 12 and 1 <= mday <= 31 and 0 <= hour <= 24 and 0 <= min <= 59 and 0 <= sec <= 61: 48 return timegm(tt) 49 else: 50 return None 51 52 53 DAYS = ['Mon', 54 'Tue', 55 'Wed', 56 'Thu', 57 'Fri', 58 'Sat', 59 'Sun'] 60 MONTHS = ['Jan', 61 'Feb', 62 'Mar', 63 'Apr', 64 'May', 65 'Jun', 66 'Jul', 67 'Aug', 68 'Sep', 69 'Oct', 70 'Nov', 71 'Dec'] 72 MONTHS_LOWER = [] 73 for month in MONTHS: 74 MONTHS_LOWER.append(month.lower()) 75 76 def time2isoz(t = None): 77 if t is None: 78 t = time.time() 79 year, mon, mday, hour, min, sec = time.gmtime(t)[:6] 80 return '%04d-%02d-%02d %02d:%02d:%02dZ' % (year, 81 mon, 82 mday, 83 hour, 84 min, 85 sec) 86 87 88 def time2netscape(t = None): 89 if t is None: 90 t = time.time() 91 year, mon, mday, hour, min, sec, wday = time.gmtime(t)[:7] 92 return '%s %02d-%s-%04d %02d:%02d:%02d GMT' % (DAYS[wday], 93 mday, 94 MONTHS[mon - 1], 95 year, 96 hour, 97 min, 98 sec) 99 100 101 UTC_ZONES = {'GMT': None, 102 'UTC': None, 103 'UT': None, 104 'Z': None} 105 TIMEZONE_RE = re.compile('^([-+])?(\\d\\d?):?(\\d\\d)?$') 106 107 def offset_from_tz_string(tz): 108 offset = None 109 if tz in UTC_ZONES: 110 offset = 0 111 else: 112 m = TIMEZONE_RE.search(tz) 113 if m: 114 offset = 3600 * int(m.group(2)) 115 if m.group(3): 116 offset = offset + 60 * int(m.group(3)) 117 if m.group(1) == '-': 118 offset = -offset 119 return offset 120 121 122 def _str2time(day, mon, yr, hr, min, sec, tz): 123 try: 124 mon = MONTHS_LOWER.index(mon.lower()) + 1 125 except ValueError: 126 try: 127 imon = int(mon) 128 except ValueError: 129 return 130 131 if 1 <= imon <= 12: 132 mon = imon 133 else: 134 return 135 136 if hr is None: 137 hr = 0 138 if min is None: 139 min = 0 140 if sec is None: 141 sec = 0 142 yr = int(yr) 143 day = int(day) 144 hr = int(hr) 145 min = int(min) 146 sec = int(sec) 147 if yr < 1000: 148 cur_yr = time.localtime(time.time())[0] 149 m = cur_yr % 100 150 tmp = yr 151 yr = yr + cur_yr - m 152 m = m - tmp 153 if abs(m) > 50: 154 if m > 0: 155 yr = yr + 100 156 else: 157 yr = yr - 100 158 t = _timegm((yr, 159 mon, 160 day, 161 hr, 162 min, 163 sec, 164 tz)) 165 if t is not None: 166 if tz is None: 167 tz = 'UTC' 168 tz = tz.upper() 169 offset = offset_from_tz_string(tz) 170 if offset is None: 171 return 172 t = t - offset 173 return t 174 175 176 STRICT_DATE_RE = re.compile('^[SMTWF][a-z][a-z], (\\d\\d) ([JFMASOND][a-z][a-z]) (\\d\\d\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d) GMT$') 177 WEEKDAY_RE = re.compile('^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\\s*', re.I) 178 LOOSE_HTTP_DATE_RE = re.compile('^\n (\\d\\d?) # day\n (?:\\s+|[-\\/])\n (\\w+) # month\n (?:\\s+|[-\\/])\n (\\d+) # year\n (?:\n (?:\\s+|:) # separator before clock\n (\\d\\d?):(\\d\\d) # hour:min\n (?::(\\d\\d))? # optional seconds\n )? # optional clock\n \\s*\n ([-+]?\\d{2,4}|(?![APap][Mm]\\b)[A-Za-z]+)? # timezone\n \\s*\n (?:\\(\\w+\\))? # ASCII representation of timezone in parens.\n \\s*$', re.X) 179 180 def http2time(text): 181 m = STRICT_DATE_RE.search(text) 182 if m: 183 g = m.groups() 184 mon = MONTHS_LOWER.index(g[1].lower()) + 1 185 tt = (int(g[2]), 186 mon, 187 int(g[0]), 188 int(g[3]), 189 int(g[4]), 190 float(g[5])) 191 return _timegm(tt) 192 text = text.lstrip() 193 text = WEEKDAY_RE.sub('', text, 1) 194 day, mon, yr, hr, min, sec, tz = [None] * 7 195 m = LOOSE_HTTP_DATE_RE.search(text) 196 if m is not None: 197 day, mon, yr, hr, min, sec, tz = m.groups() 198 else: 199 return 200 return _str2time(day, mon, yr, hr, min, sec, tz) 201 202 203 ISO_DATE_RE = re.compile('^\n (\\d{4}) # year\n [-\\/]?\n (\\d\\d?) # numerical month\n [-\\/]?\n (\\d\\d?) # day\n (?:\n (?:\\s+|[-:Tt]) # separator before clock\n (\\d\\d?):?(\\d\\d) # hour:min\n (?::?(\\d\\d(?:\\.\\d*)?))? # optional seconds (and fractional)\n )? # optional clock\n \\s*\n ([-+]?\\d\\d?:?(:?\\d\\d)?\n |Z|z)? # timezone (Z is "zero meridian", i.e. GMT)\n \\s*$', re.X) 204 205 def iso2time(text): 206 text = text.lstrip() 207 day, mon, yr, hr, min, sec, tz = [None] * 7 208 m = ISO_DATE_RE.search(text) 209 if m is not None: 210 yr, mon, day, hr, min, sec, tz, _ = m.groups() 211 else: 212 return 213 return _str2time(day, mon, yr, hr, min, sec, tz) 214 215 216 def unmatched(match): 217 start, end = match.span(0) 218 return match.string[:start] + match.string[end:] 219 220 221 HEADER_TOKEN_RE = re.compile('^\\s*([^=\\s;,]+)') 222 HEADER_QUOTED_VALUE_RE = re.compile('^\\s*=\\s*\\"([^\\"\\\\]*(?:\\\\.[^\\"\\\\]*)*)\\"') 223 HEADER_VALUE_RE = re.compile('^\\s*=\\s*([^\\s;,]*)') 224 HEADER_ESCAPE_RE = re.compile('\\\\(.)') 225 226 def split_header_words(header_values): 227 result = [] 228 for text in header_values: 229 orig_text = text 230 pairs = [] 231 while text: 232 m = HEADER_TOKEN_RE.search(text) 233 if m: 234 text = unmatched(m) 235 name = m.group(1) 236 m = HEADER_QUOTED_VALUE_RE.search(text) 237 if m: 238 text = unmatched(m) 239 value = m.group(1) 240 value = HEADER_ESCAPE_RE.sub('\\1', value) 241 else: 242 m = HEADER_VALUE_RE.search(text) 243 if m: 244 text = unmatched(m) 245 value = m.group(1) 246 value = value.rstrip() 247 else: 248 value = None 249 pairs.append((name, value)) 250 elif text.lstrip().startswith(','): 251 text = text.lstrip()[1:] 252 if pairs: 253 result.append(pairs) 254 pairs = [] 255 else: 256 non_junk, nr_junk_chars = re.subn('^[=\\s;]*', '', text) 257 text = non_junk 258 259 if pairs: 260 result.append(pairs) 261 262 return result 263 264 265 HEADER_JOIN_ESCAPE_RE = re.compile('([\\"\\\\])') 266 267 def join_header_words(lists): 268 headers = [] 269 for pairs in lists: 270 attr = [] 271 for k, v in pairs: 272 if v is not None: 273 if not re.search('^\\w+$', v): 274 v = HEADER_JOIN_ESCAPE_RE.sub('\\\\\\1', v) 275 v = '"%s"' % v 276 k = '%s=%s' % (k, v) 277 attr.append(k) 278 279 if attr: 280 headers.append('; '.join(attr)) 281 282 return ', '.join(headers) 283 284 285 def _strip_quotes(text): 286 if text.startswith('"'): 287 text = text[1:] 288 if text.endswith('"'): 289 text = text[:-1] 290 return text 291 292 293 def parse_ns_headers(ns_headers): 294 known_attrs = ('expires', 'domain', 'path', 'secure', 'version', 'port', 'max-age') 295 result = [] 296 for ns_header in ns_headers: 297 pairs = [] 298 version_set = False 299 for ii, param in enumerate(re.split(';\\s*', ns_header)): 300 param = param.rstrip() 301 if param == '': 302 continue 303 if '=' not in param: 304 k, v = param, None 305 else: 306 k, v = re.split('\\s*=\\s*', param, 1) 307 k = k.lstrip() 308 if ii != 0: 309 lc = k.lower() 310 if lc in known_attrs: 311 k = lc 312 if k == 'version': 313 v = _strip_quotes(v) 314 version_set = True 315 if k == 'expires': 316 v = http2time(_strip_quotes(v)) 317 pairs.append((k, v)) 318 319 if pairs: 320 if not version_set: 321 pairs.append(('version', '0')) 322 result.append(pairs) 323 324 return result 325 326 327 IPV4_RE = re.compile('\\.\\d+$') 328 329 def is_HDN(text): 330 if IPV4_RE.search(text): 331 return False 332 if text == '': 333 return False 334 if text[0] == '.' or text[-1] == '.': 335 return False 336 return True 337 338 339 def domain_match(A, B): 340 A = A.lower() 341 B = B.lower() 342 if A == B: 343 return True 344 if not is_HDN(A): 345 return False 346 i = A.rfind(B) 347 if i == -1 or i == 0: 348 return False 349 if not B.startswith('.'): 350 return False 351 if not is_HDN(B[1:]): 352 return False 353 return True 354 355 356 def liberal_is_HDN(text): 357 if IPV4_RE.search(text): 358 return False 359 return True 360 361 362 def user_domain_match(A, B): 363 A = A.lower() 364 B = B.lower() 365 if not (liberal_is_HDN(A) and liberal_is_HDN(B)): 366 if A == B: 367 return True 368 return False 369 initial_dot = B.startswith('.') 370 if initial_dot and A.endswith(B): 371 return True 372 if not initial_dot and A == B: 373 return True 374 return False 375 376 377 cut_port_re = re.compile(':\\d+$') 378 379 def request_host(request): 380 url = request.get_full_url() 381 host = urlparse.urlparse(url)[1] 382 if host == '': 383 host = request.get_header('Host', '') 384 host = cut_port_re.sub('', host, 1) 385 return host.lower() 386 387 388 def eff_request_host(request): 389 erhn = req_host = request_host(request) 390 if req_host.find('.') == -1 and not IPV4_RE.search(req_host): 391 erhn = req_host + '.local' 392 return (req_host, erhn) 393 394 395 def request_path(request): 396 url = request.get_full_url() 397 parts = urlparse.urlsplit(url) 398 path = escape_path(parts.path) 399 if not path.startswith('/'): 400 path = '/' + path 401 return path 402 403 404 def request_port(request): 405 host = request.get_host() 406 i = host.find(':') 407 if i >= 0: 408 port = host[i + 1:] 409 try: 410 int(port) 411 except ValueError: 412 _debug("nonnumeric port: '%s'", port) 413 return None 414 415 else: 416 port = DEFAULT_HTTP_PORT 417 return port 418 419 420 HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()" 421 ESCAPED_CHAR_RE = re.compile('%([0-9a-fA-F][0-9a-fA-F])') 422 423 def uppercase_escaped_char(match): 424 return '%%%s' % match.group(1).upper() 425 426 427 def escape_path(path): 428 if isinstance(path, unicode): 429 path = path.encode('utf-8') 430 path = urllib.quote(path, HTTP_PATH_SAFE) 431 path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path) 432 return path 433 434 435 def reach(h): 436 i = h.find('.') 437 if i >= 0: 438 b = h[i + 1:] 439 i = b.find('.') 440 if is_HDN(h) and (i >= 0 or b == 'local'): 441 return '.' + b 442 return h 443 444 445 def is_third_party(request): 446 req_host = request_host(request) 447 if not domain_match(req_host, reach(request.get_origin_req_host())): 448 return True 449 else: 450 return False 451 452 453 class Cookie(): 454 455 def __init__(self, version, name, value, port, port_specified, domain, domain_specified, domain_initial_dot, path, path_specified, secure, expires, discard, comment, comment_url, rest, rfc2109 = False): 456 if version is not None: 457 version = int(version) 458 if expires is not None: 459 expires = int(expires) 460 if port is None and port_specified is True: 461 raise ValueError('if port is None, port_specified must be false') 462 self.version = version 463 self.name = name 464 self.value = value 465 self.port = port 466 self.port_specified = port_specified 467 self.domain = domain.lower() 468 self.domain_specified = domain_specified 469 self.domain_initial_dot = domain_initial_dot 470 self.path = path 471 self.path_specified = path_specified 472 self.secure = secure 473 self.expires = expires 474 self.discard = discard 475 self.comment = comment 476 self.comment_url = comment_url 477 self.rfc2109 = rfc2109 478 self._rest = copy.copy(rest) 479 480 def has_nonstandard_attr(self, name): 481 return name in self._rest 482 483 def get_nonstandard_attr(self, name, default = None): 484 return self._rest.get(name, default) 485 486 def set_nonstandard_attr(self, name, value): 487 self._rest[name] = value 488 489 def is_expired(self, now = None): 490 if now is None: 491 now = time.time() 492 if self.expires is not None and self.expires <= now: 493 return True 494 return False 495 496 def __str__(self): 497 if self.port is None: 498 p = '' 499 else: 500 p = ':' + self.port 501 limit = self.domain + p + self.path 502 if self.value is not None: 503 namevalue = '%s=%s' % (self.name, self.value) 504 else: 505 namevalue = self.name 506 return '<Cookie %s for %s>' % (namevalue, limit) 507 508 def __repr__(self): 509 args = [] 510 for name in ('version', 'name', 'value', 'port', 'port_specified', 'domain', 'domain_specified', 'domain_initial_dot', 'path', 'path_specified', 'secure', 'expires', 'discard', 'comment', 'comment_url'): 511 attr = getattr(self, name) 512 args.append('%s=%s' % (name, repr(attr))) 513 514 args.append('rest=%s' % repr(self._rest)) 515 args.append('rfc2109=%s' % repr(self.rfc2109)) 516 return 'Cookie(%s)' % ', '.join(args) 517 518 519 class CookiePolicy(): 520 521 def set_ok(self, cookie, request): 522 raise NotImplementedError() 523 524 def return_ok(self, cookie, request): 525 raise NotImplementedError() 526 527 def domain_return_ok(self, domain, request): 528 return True 529 530 def path_return_ok(self, path, request): 531 return True 532 533 534 class DefaultCookiePolicy(CookiePolicy): 535 DomainStrictNoDots = 1 536 DomainStrictNonDomain = 2 537 DomainRFC2965Match = 4 538 DomainLiberal = 0 539 DomainStrict = DomainStrictNoDots | DomainStrictNonDomain 540 541 def __init__(self, blocked_domains = None, allowed_domains = None, netscape = True, rfc2965 = False, rfc2109_as_netscape = None, hide_cookie2 = False, strict_domain = False, strict_rfc2965_unverifiable = True, strict_ns_unverifiable = False, strict_ns_domain = DomainLiberal, strict_ns_set_initial_dollar = False, strict_ns_set_path = False): 542 self.netscape = netscape 543 self.rfc2965 = rfc2965 544 self.rfc2109_as_netscape = rfc2109_as_netscape 545 self.hide_cookie2 = hide_cookie2 546 self.strict_domain = strict_domain 547 self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable 548 self.strict_ns_unverifiable = strict_ns_unverifiable 549 self.strict_ns_domain = strict_ns_domain 550 self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar 551 self.strict_ns_set_path = strict_ns_set_path 552 if blocked_domains is not None: 553 self._blocked_domains = tuple(blocked_domains) 554 else: 555 self._blocked_domains = () 556 if allowed_domains is not None: 557 allowed_domains = tuple(allowed_domains) 558 self._allowed_domains = allowed_domains 559 560 def blocked_domains(self): 561 return self._blocked_domains 562 563 def set_blocked_domains(self, blocked_domains): 564 self._blocked_domains = tuple(blocked_domains) 565 566 def is_blocked(self, domain): 567 for blocked_domain in self._blocked_domains: 568 if user_domain_match(domain, blocked_domain): 569 return True 570 571 return False 572 573 def allowed_domains(self): 574 return self._allowed_domains 575 576 def set_allowed_domains(self, allowed_domains): 577 if allowed_domains is not None: 578 allowed_domains = tuple(allowed_domains) 579 self._allowed_domains = allowed_domains 580 581 def is_not_allowed(self, domain): 582 if self._allowed_domains is None: 583 return False 584 for allowed_domain in self._allowed_domains: 585 if user_domain_match(domain, allowed_domain): 586 return False 587 588 return True 589 590 def set_ok(self, cookie, request): 591 _debug(' - checking cookie %s=%s', cookie.name, cookie.value) 592 for n in ('version', 'verifiability', 'name', 'path', 'domain', 'port'): 593 fn_name = 'set_ok_' + n 594 fn = getattr(self, fn_name) 595 if not fn(cookie, request): 596 return False 597 598 return True 599 600 def set_ok_version(self, cookie, request): 601 if cookie.version is None: 602 _debug(' Set-Cookie2 without version attribute (%s=%s)', cookie.name, cookie.value) 603 return False 604 if cookie.version > 0 and not self.rfc2965: 605 _debug(' RFC 2965 cookies are switched off') 606 return False 607 if cookie.version == 0 and not self.netscape: 608 _debug(' Netscape cookies are switched off') 609 return False 610 return True 611 612 def set_ok_verifiability(self, cookie, request): 613 if request.is_unverifiable() and is_third_party(request): 614 if cookie.version > 0 and self.strict_rfc2965_unverifiable: 615 _debug(' third-party RFC 2965 cookie during unverifiable transaction') 616 return False 617 if cookie.version == 0 and self.strict_ns_unverifiable: 618 _debug(' third-party Netscape cookie during unverifiable transaction') 619 return False 620 return True 621 622 def set_ok_name(self, cookie, request): 623 if cookie.version == 0 and self.strict_ns_set_initial_dollar and cookie.name.startswith('$'): 624 _debug(" illegal name (starts with '$'): '%s'", cookie.name) 625 return False 626 return True 627 628 def set_ok_path(self, cookie, request): 629 if cookie.path_specified: 630 req_path = request_path(request) 631 if (cookie.version > 0 or cookie.version == 0 and self.strict_ns_set_path) and not req_path.startswith(cookie.path): 632 _debug(' path attribute %s is not a prefix of request path %s', cookie.path, req_path) 633 return False 634 return True 635 636 def set_ok_domain(self, cookie, request): 637 if self.is_blocked(cookie.domain): 638 _debug(' domain %s is in user block-list', cookie.domain) 639 return False 640 if self.is_not_allowed(cookie.domain): 641 _debug(' domain %s is not in user allow-list', cookie.domain) 642 return False 643 if cookie.domain_specified: 644 req_host, erhn = eff_request_host(request) 645 domain = cookie.domain 646 if self.strict_domain and domain.count('.') >= 2: 647 i = domain.rfind('.') 648 j = domain.rfind('.', 0, i) 649 if j == 0: 650 tld = domain[i + 1:] 651 sld = domain[j + 1:i] 652 if sld.lower() in ('co', 'ac', 'com', 'edu', 'org', 'net', 'gov', 'mil', 'int', 'aero', 'biz', 'cat', 'coop', 'info', 'jobs', 'mobi', 'museum', 'name', 'pro', 'travel', 'eu') and len(tld) == 2: 653 _debug(' country-code second level domain %s', domain) 654 return False 655 if domain.startswith('.'): 656 undotted_domain = domain[1:] 657 else: 658 undotted_domain = domain 659 embedded_dots = undotted_domain.find('.') >= 0 660 if not embedded_dots and domain != '.local': 661 _debug(' non-local domain %s contains no embedded dot', domain) 662 return False 663 if cookie.version == 0 and not erhn.endswith(domain) and not erhn.startswith('.'): 664 if not ('.' + erhn).endswith(domain): 665 _debug(' effective request-host %s (even with added initial dot) does not end end with %s', erhn, domain) 666 return False 667 if cookie.version > 0 or self.strict_ns_domain & self.DomainRFC2965Match: 668 if not domain_match(erhn, domain): 669 _debug(' effective request-host %s does not domain-match %s', erhn, domain) 670 return False 671 if cookie.version > 0 or self.strict_ns_domain & self.DomainStrictNoDots: 672 host_prefix = req_host[:-len(domain)] 673 if host_prefix.find('.') >= 0 and not IPV4_RE.search(req_host): 674 _debug(' host prefix %s for domain %s contains a dot', host_prefix, domain) 675 return False 676 return True 677 678 def set_ok_port(self, cookie, request): 679 if cookie.port_specified: 680 req_port = request_port(request) 681 if req_port is None: 682 req_port = '80' 683 else: 684 req_port = str(req_port) 685 for p in cookie.port.split(','): 686 try: 687 int(p) 688 except ValueError: 689 _debug(' bad port %s (not numeric)', p) 690 return False 691 692 if p == req_port: 693 break 694 else: 695 _debug(' request port (%s) not found in %s', req_port, cookie.port) 696 return False 697 698 return True 699 700 def return_ok(self, cookie, request): 701 _debug(' - checking cookie %s=%s', cookie.name, cookie.value) 702 for n in ('version', 'verifiability', 'secure', 'expires', 'port', 'domain'): 703 fn_name = 'return_ok_' + n 704 fn = getattr(self, fn_name) 705 if not fn(cookie, request): 706 return False 707 708 return True 709 710 def return_ok_version(self, cookie, request): 711 if cookie.version > 0 and not self.rfc2965: 712 _debug(' RFC 2965 cookies are switched off') 713 return False 714 if cookie.version == 0 and not self.netscape: 715 _debug(' Netscape cookies are switched off') 716 return False 717 return True 718 719 def return_ok_verifiability(self, cookie, request): 720 if request.is_unverifiable() and is_third_party(request): 721 if cookie.version > 0 and self.strict_rfc2965_unverifiable: 722 _debug(' third-party RFC 2965 cookie during unverifiable transaction') 723 return False 724 if cookie.version == 0 and self.strict_ns_unverifiable: 725 _debug(' third-party Netscape cookie during unverifiable transaction') 726 return False 727 return True 728 729 def return_ok_secure(self, cookie, request): 730 if cookie.secure and request.get_type() != 'https': 731 _debug(' secure cookie with non-secure request') 732 return False 733 return True 734 735 def return_ok_expires(self, cookie, request): 736 if cookie.is_expired(self._now): 737 _debug(' cookie expired') 738 return False 739 return True 740 741 def return_ok_port(self, cookie, request): 742 if cookie.port: 743 req_port = request_port(request) 744 if req_port is None: 745 req_port = '80' 746 for p in cookie.port.split(','): 747 if p == req_port: 748 break 749 else: 750 _debug(' request port %s does not match cookie port %s', req_port, cookie.port) 751 return False 752 753 return True 754 755 def return_ok_domain(self, cookie, request): 756 req_host, erhn = eff_request_host(request) 757 domain = cookie.domain 758 if cookie.version == 0 and self.strict_ns_domain & self.DomainStrictNonDomain and not cookie.domain_specified and domain != erhn: 759 _debug(' cookie with unspecified domain does not string-compare equal to request domain') 760 return False 761 if cookie.version > 0 and not domain_match(erhn, domain): 762 _debug(' effective request-host name %s does not domain-match RFC 2965 cookie domain %s', erhn, domain) 763 return False 764 if cookie.version == 0 and not ('.' + erhn).endswith(domain): 765 _debug(' request-host %s does not match Netscape cookie domain %s', req_host, domain) 766 return False 767 return True 768 769 def domain_return_ok(self, domain, request): 770 req_host, erhn = eff_request_host(request) 771 if not req_host.startswith('.'): 772 req_host = '.' + req_host 773 if not erhn.startswith('.'): 774 erhn = '.' + erhn 775 if not (req_host.endswith(domain) or erhn.endswith(domain)): 776 return False 777 if self.is_blocked(domain): 778 _debug(' domain %s is in user block-list', domain) 779 return False 780 if self.is_not_allowed(domain): 781 _debug(' domain %s is not in user allow-list', domain) 782 return False 783 return True 784 785 def path_return_ok(self, path, request): 786 _debug('- checking cookie path=%s', path) 787 req_path = request_path(request) 788 if not req_path.startswith(path): 789 _debug(' %s does not path-match %s', req_path, path) 790 return False 791 return True 792 793 794 def vals_sorted_by_key(adict): 795 keys = adict.keys() 796 keys.sort() 797 return map(adict.get, keys) 798 799 800 def deepvalues(mapping): 801 values = vals_sorted_by_key(mapping) 802 for obj in values: 803 mapping = False 804 try: 805 obj.items 806 except AttributeError: 807 pass 808 else: 809 mapping = True 810 for subobj in deepvalues(obj): 811 yield subobj 812 813 if not mapping: 814 yield obj 815 816 817 class Absent(): 818 pass 819 820 821 class CookieJar(): 822 non_word_re = re.compile('\\W') 823 quote_re = re.compile('([\\"\\\\])') 824 strict_domain_re = re.compile('\\.?[^.]*') 825 domain_re = re.compile('[^.]*') 826 dots_re = re.compile('^\\.+') 827 magic_re = '^\\#LWP-Cookies-(\\d+\\.\\d+)' 828 829 def __init__(self, policy = None): 830 if policy is None: 831 policy = DefaultCookiePolicy() 832 self._policy = policy 833 self._cookies_lock = _threading.RLock() 834 self._cookies = {} 835 836 def set_policy(self, policy): 837 self._policy = policy 838 839 def _cookies_for_domain(self, domain, request): 840 cookies = [] 841 if not self._policy.domain_return_ok(domain, request): 842 return [] 843 _debug('Checking %s for cookies to return', domain) 844 cookies_by_path = self._cookies[domain] 845 for path in cookies_by_path.keys(): 846 if not self._policy.path_return_ok(path, request): 847 continue 848 cookies_by_name = cookies_by_path[path] 849 for cookie in cookies_by_name.values(): 850 if not self._policy.return_ok(cookie, request): 851 _debug(' not returning cookie') 852 continue 853 _debug(" it's a match") 854 cookies.append(cookie) 855 856 return cookies 857 858 def _cookies_for_request(self, request): 859 cookies = [] 860 for domain in self._cookies.keys(): 861 cookies.extend(self._cookies_for_domain(domain, request)) 862 863 return cookies 864 865 def _cookie_attrs(self, cookies): 866 cookies.sort(key=lambda arg: len(arg.path), reverse=True) 867 version_set = False 868 attrs = [] 869 for cookie in cookies: 870 version = cookie.version 871 if not version_set: 872 version_set = True 873 if version > 0: 874 attrs.append('$Version=%s' % version) 875 if cookie.value is not None and self.non_word_re.search(cookie.value) and version > 0: 876 value = self.quote_re.sub('\\\\\\1', cookie.value) 877 else: 878 value = cookie.value 879 if cookie.value is None: 880 attrs.append(cookie.name) 881 else: 882 attrs.append('%s=%s' % (cookie.name, value)) 883 if version > 0: 884 if cookie.path_specified: 885 attrs.append('$Path="%s"' % cookie.path) 886 if cookie.domain.startswith('.'): 887 domain = cookie.domain 888 if not cookie.domain_initial_dot and domain.startswith('.'): 889 domain = domain[1:] 890 attrs.append('$Domain="%s"' % domain) 891 if cookie.port is not None: 892 p = '$Port' 893 if cookie.port_specified: 894 p = p + '="%s"' % cookie.port 895 attrs.append(p) 896 897 return attrs 898 899 def add_cookie_header(self, request): 900 _debug('add_cookie_header') 901 self._cookies_lock.acquire() 902 try: 903 self._policy._now = self._now = int(time.time()) 904 cookies = self._cookies_for_request(request) 905 attrs = self._cookie_attrs(cookies) 906 if attrs: 907 if not request.has_header('Cookie'): 908 request.add_unredirected_header('Cookie', '; '.join(attrs)) 909 if self._policy.rfc2965 and not self._policy.hide_cookie2 and not request.has_header('Cookie2'): 910 for cookie in cookies: 911 if cookie.version != 1: 912 request.add_unredirected_header('Cookie2', '$Version="1"') 913 break 914 915 finally: 916 self._cookies_lock.release() 917 918 self.clear_expired_cookies() 919 920 def _normalized_cookie_tuples(self, attrs_set): 921 cookie_tuples = [] 922 boolean_attrs = ('discard', 'secure') 923 value_attrs = ('version', 'expires', 'max-age', 'domain', 'path', 'port', 'comment', 'commenturl') 924 for cookie_attrs in attrs_set: 925 name, value = cookie_attrs[0] 926 max_age_set = False 927 bad_cookie = False 928 standard = {} 929 rest = {} 930 for k, v in cookie_attrs[1:]: 931 lc = k.lower() 932 if lc in value_attrs or lc in boolean_attrs: 933 k = lc 934 if k in boolean_attrs and v is None: 935 v = True 936 if k in standard: 937 continue 938 if k == 'domain': 939 if v is None: 940 _debug(' missing value for domain attribute') 941 bad_cookie = True 942 break 943 v = v.lower() 944 if k == 'expires': 945 if max_age_set: 946 continue 947 if v is None: 948 _debug(' missing or invalid value for expires attribute: treating as session cookie') 949 continue 950 if k == 'max-age': 951 max_age_set = True 952 try: 953 v = int(v) 954 except ValueError: 955 _debug(' missing or invalid (non-numeric) value for max-age attribute') 956 bad_cookie = True 957 break 958 959 k = 'expires' 960 v = self._now + v 961 if k in value_attrs or k in boolean_attrs: 962 if v is None and k not in ('port', 'comment', 'commenturl'): 963 _debug(' missing value for %s attribute' % k) 964 bad_cookie = True 965 break 966 standard[k] = v 967 else: 968 rest[k] = v 969 970 if bad_cookie: 971 continue 972 cookie_tuples.append((name, 973 value, 974 standard, 975 rest)) 976 977 return cookie_tuples 978 979 def _cookie_from_cookie_tuple(self, tup, request): 980 name, value, standard, rest = tup 981 domain = standard.get('domain', Absent) 982 path = standard.get('path', Absent) 983 port = standard.get('port', Absent) 984 expires = standard.get('expires', Absent) 985 version = standard.get('version', None) 986 if version is not None: 987 try: 988 version = int(version) 989 except ValueError: 990 return 991 992 secure = standard.get('secure', False) 993 discard = standard.get('discard', False) 994 comment = standard.get('comment', None) 995 comment_url = standard.get('commenturl', None) 996 if path is not Absent and path != '': 997 path_specified = True 998 path = escape_path(path) 999 else: 1000 path_specified = False 1001 path = request_path(request) 1002 i = path.rfind('/') 1003 if i != -1: 1004 if version == 0: 1005 path = path[:i] 1006 else: 1007 path = path[:i + 1] 1008 if len(path) == 0: 1009 path = '/' 1010 domain_specified = domain is not Absent 1011 domain_initial_dot = False 1012 if domain_specified: 1013 domain_initial_dot = bool(domain.startswith('.')) 1014 if domain is Absent: 1015 req_host, erhn = eff_request_host(request) 1016 domain = erhn 1017 elif not domain.startswith('.'): 1018 domain = '.' + domain 1019 port_specified = False 1020 if port is not Absent: 1021 if port is None: 1022 port = request_port(request) 1023 else: 1024 port_specified = True 1025 port = re.sub('\\s+', '', port) 1026 else: 1027 port = None 1028 if expires is Absent: 1029 expires = None 1030 discard = True 1031 elif expires <= self._now: 1032 try: 1033 self.clear(domain, path, name) 1034 except KeyError: 1035 pass 1036 1037 _debug("Expiring cookie, domain='%s', path='%s', name='%s'", domain, path, name) 1038 return 1039 return Cookie(version, name, value, port, port_specified, domain, domain_specified, domain_initial_dot, path, path_specified, secure, expires, discard, comment, comment_url, rest) 1040 1041 def _cookies_from_attrs_set(self, attrs_set, request): 1042 cookie_tuples = self._normalized_cookie_tuples(attrs_set) 1043 cookies = [] 1044 for tup in cookie_tuples: 1045 cookie = self._cookie_from_cookie_tuple(tup, request) 1046 if cookie: 1047 cookies.append(cookie) 1048 1049 return cookies 1050 1051 def _process_rfc2109_cookies(self, cookies): 1052 rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None) 1053 if rfc2109_as_ns is None: 1054 rfc2109_as_ns = not self._policy.rfc2965 1055 for cookie in cookies: 1056 if cookie.version == 1: 1057 cookie.rfc2109 = True 1058 if rfc2109_as_ns: 1059 cookie.version = 0 1060 1061 def make_cookies(self, response, request): 1062 headers = response.info() 1063 rfc2965_hdrs = headers.getheaders('Set-Cookie2') 1064 ns_hdrs = headers.getheaders('Set-Cookie') 1065 rfc2965 = self._policy.rfc2965 1066 netscape = self._policy.netscape 1067 if not rfc2965_hdrs and not ns_hdrs or not ns_hdrs and not rfc2965 or not rfc2965_hdrs and not netscape or not netscape and not rfc2965: 1068 return [] 1069 try: 1070 cookies = self._cookies_from_attrs_set(split_header_words(rfc2965_hdrs), request) 1071 except Exception: 1072 _warn_unhandled_exception() 1073 cookies = [] 1074 1075 if ns_hdrs and netscape: 1076 try: 1077 ns_cookies = self._cookies_from_attrs_set(parse_ns_headers(ns_hdrs), request) 1078 except Exception: 1079 _warn_unhandled_exception() 1080 ns_cookies = [] 1081 1082 self._process_rfc2109_cookies(ns_cookies) 1083 if rfc2965: 1084 lookup = {} 1085 for cookie in cookies: 1086 lookup[cookie.domain, cookie.path, cookie.name] = None 1087 1088 def no_matching_rfc2965(ns_cookie, lookup = lookup): 1089 key = (ns_cookie.domain, ns_cookie.path, ns_cookie.name) 1090 return key not in lookup 1091 1092 ns_cookies = filter(no_matching_rfc2965, ns_cookies) 1093 if ns_cookies: 1094 cookies.extend(ns_cookies) 1095 return cookies 1096 1097 def set_cookie_if_ok(self, cookie, request): 1098 self._cookies_lock.acquire() 1099 try: 1100 self._policy._now = self._now = int(time.time()) 1101 if self._policy.set_ok(cookie, request): 1102 self.set_cookie(cookie) 1103 finally: 1104 self._cookies_lock.release() 1105 1106 def set_cookie(self, cookie): 1107 c = self._cookies 1108 self._cookies_lock.acquire() 1109 try: 1110 if cookie.domain not in c: 1111 c[cookie.domain] = {} 1112 c2 = c[cookie.domain] 1113 if cookie.path not in c2: 1114 c2[cookie.path] = {} 1115 c3 = c2[cookie.path] 1116 c3[cookie.name] = cookie 1117 finally: 1118 self._cookies_lock.release() 1119 1120 def extract_cookies(self, response, request): 1121 _debug('extract_cookies: %s', response.info()) 1122 self._cookies_lock.acquire() 1123 try: 1124 self._policy._now = self._now = int(time.time()) 1125 for cookie in self.make_cookies(response, request): 1126 if self._policy.set_ok(cookie, request): 1127 _debug(' setting cookie: %s', cookie) 1128 self.set_cookie(cookie) 1129 1130 finally: 1131 self._cookies_lock.release() 1132 1133 def clear(self, domain = None, path = None, name = None): 1134 if name is not None: 1135 if domain is None or path is None: 1136 raise ValueError('domain and path must be given to remove a cookie by name') 1137 del self._cookies[domain][path][name] 1138 elif path is not None: 1139 if domain is None: 1140 raise ValueError('domain must be given to remove cookies by path') 1141 del self._cookies[domain][path] 1142 elif domain is not None: 1143 del self._cookies[domain] 1144 else: 1145 self._cookies = {} 1146 1147 def clear_session_cookies(self): 1148 self._cookies_lock.acquire() 1149 try: 1150 for cookie in self: 1151 if cookie.discard: 1152 self.clear(cookie.domain, cookie.path, cookie.name) 1153 1154 finally: 1155 self._cookies_lock.release() 1156 1157 def clear_expired_cookies(self): 1158 self._cookies_lock.acquire() 1159 try: 1160 now = time.time() 1161 for cookie in self: 1162 if cookie.is_expired(now): 1163 self.clear(cookie.domain, cookie.path, cookie.name) 1164 1165 finally: 1166 self._cookies_lock.release() 1167 1168 def __iter__(self): 1169 return deepvalues(self._cookies) 1170 1171 def __len__(self): 1172 i = 0 1173 for cookie in self: 1174 i = i + 1 1175 1176 return i 1177 1178 def __repr__(self): 1179 r = [] 1180 for cookie in self: 1181 r.append(repr(cookie)) 1182 1183 return '<%s[%s]>' % (self.__class__, ', '.join(r)) 1184 1185 def __str__(self): 1186 r = [] 1187 for cookie in self: 1188 r.append(str(cookie)) 1189 1190 return '<%s[%s]>' % (self.__class__, ', '.join(r)) 1191 1192 1193 class LoadError(IOError): 1194 pass 1195 1196 1197 class FileCookieJar(CookieJar): 1198 1199 def __init__(self, filename = None, delayload = False, policy = None): 1200 CookieJar.__init__(self, policy) 1201 if filename is not None: 1202 try: 1203 filename + '' 1204 except: 1205 raise ValueError('filename must be string-like') 1206 1207 self.filename = filename 1208 self.delayload = bool(delayload) 1209 1210 def save(self, filename = None, ignore_discard = False, ignore_expires = False): 1211 raise NotImplementedError() 1212 1213 def load(self, filename = None, ignore_discard = False, ignore_expires = False): 1214 if filename is None: 1215 if self.filename is not None: 1216 filename = self.filename 1217 else: 1218 raise ValueError(MISSING_FILENAME_TEXT) 1219 f = open(filename) 1220 try: 1221 self._really_load(f, filename, ignore_discard, ignore_expires) 1222 finally: 1223 f.close() 1224 1225 def revert(self, filename = None, ignore_discard = False, ignore_expires = False): 1226 if filename is None: 1227 if self.filename is not None: 1228 filename = self.filename 1229 else: 1230 raise ValueError(MISSING_FILENAME_TEXT) 1231 self._cookies_lock.acquire() 1232 try: 1233 old_state = copy.deepcopy(self._cookies) 1234 self._cookies = {} 1235 try: 1236 self.load(filename, ignore_discard, ignore_expires) 1237 except (LoadError, IOError): 1238 self._cookies = old_state 1239 raise 1240 1241 finally: 1242 self._cookies_lock.release() 1243 1244 1245 from _LWPCookieJar import LWPCookieJar, lwp_cookie_str 1246 from _MozillaCookieJar import MozillaCookieJar