httpService.py (50785B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\script\net\httpService.py 2 from __future__ import print_function 3 import StringIO 4 import binascii 5 import cStringIO 6 import cgi 7 import logging 8 import os 9 import sys 10 import time 11 import traceback 12 import urllib 13 import uuid 14 import blue 15 import bluepy 16 import carbon.common.script.net.machobase as macho 17 import carbon.common.script.util.htmlwriter as htmlwriter 18 import evecrypto.crypto as Crypto 19 import iocp 20 import log 21 import typeutils 22 import uthread 23 import utillib 24 from caching.memoize import Memoize 25 from carbon.common.script.net import GPSExceptions 26 from carbon.common.script.net import machoNetExceptions 27 from carbon.common.script.sys import basesession 28 from carbon.common.script.sys.service import CoreService, Service 29 from carbon.common.script.sys.serviceConst import ROLE_ADMIN, ROLE_PROGRAMMER, ROLE_SERVICE, SERVICE_RUNNING, SERVICE_STOPPED, SERVICE_STOP_PENDING 30 from carbon.common.script.util.timerstuff import ClockThis 31 from carbon.common.script.util.format import PasswordString 32 from carbon.common.lib import const 33 from eveexceptions import SQLError, UserError 34 from requests import structures 35 from scriber import httputils 36 from eveprefs import prefs, boot 37 from stdlogutils import LineWrap 38 httplog = logging.getLogger('httplog') 39 svclog = logging.getLogger('svc.http') 40 DB_SETTING_CACHE_TIMEOUT = 10 41 if macho.mode == 'proxy': 42 DB_SETTING_CACHE_TIMEOUT = 60 43 44 class HeaderDict(structures.CaseInsensitiveDict): 45 46 def has_key(self, key): 47 return key in self 48 49 50 class AuthError(RuntimeError): 51 pass 52 53 54 class UnknownProtocol(RuntimeError): 55 pass 56 57 58 class UnknownVerb(RuntimeError): 59 pass 60 61 62 class Request(): 63 64 def __init__(self, httpService, ep): 65 self.httpService = httpService 66 self.ep = ep 67 self.buff = '' 68 self.start_time = time.time() 69 self.header = HeaderDict() 70 self.query = {} 71 self.form = {} 72 self.formfields = None 73 self.cookie = {} 74 self.totalBytes = 0 75 self.path = '' 76 self.full_path = '' 77 self.paths = [] 78 self.method = '' 79 self.proto = '' 80 self.args = '' 81 self.raw = '' 82 self.tunnelinfo = None 83 self.client_ip = '' 84 self.uid = '' 85 self.host = '' 86 self.session = None 87 self.node_id = 0 88 self.via_node_id = 0 89 self.macho_mode = '' 90 91 def ClientCertificate(self, key): 92 return None 93 94 def FullPath(self): 95 page = self.path + '?' 96 for k, v in self.query.iteritems(): 97 if k: 98 page += '%s=%s&' % (k, v) 99 100 return page 101 102 def Cookies(self, cookie): 103 return self.cookie[cookie] 104 105 def Form(self, element): 106 if element not in self.form: 107 return None 108 return htmlwriter.Pythonize(self.form[element]) 109 110 def FormItems(self): 111 return dict(((k, htmlwriter.Pythonize(v)) for k, v in self.form.items())) 112 113 def FormCheck(self, element): 114 if self.Form(element) == 'on': 115 return 1 116 return 0 117 118 def QueryStrings(self, raw = False): 119 ret = {} 120 for k, v in self.query.iteritems(): 121 if raw: 122 ret[k] = v 123 else: 124 ret[k] = htmlwriter.Pythonize(v) 125 126 return ret 127 128 def QueryString(self, variable, caseInsensitive = False, raw = False, default = None): 129 if caseInsensitive: 130 variable = variable.lower() 131 for k, v in self.query.iteritems(): 132 if k.lower() == variable: 133 if not raw: 134 return htmlwriter.Pythonize(v) 135 return v 136 137 elif variable in self.query: 138 v = self.query[variable] 139 if not raw: 140 return htmlwriter.Pythonize(v) 141 return v 142 return default 143 144 def ServerVariables(self, env): 145 if env == 'SERVER_SOFTWARE': 146 return 'CCP Server Pages 1.0' 147 elif env == 'SERVER_PORT': 148 return 'Is this being used?' 149 elif env in self.header: 150 return self.header[env] 151 else: 152 return None 153 154 def ReadRequest(self): 155 chunks = [self.buff] 156 while True: 157 end = chunks[-1].find('\r\n\r\n') 158 if end >= 0: 159 break 160 chunks.append(self.ep.Read()) 161 162 end += 4 163 tail = chunks[-1] 164 chunks[-1] = tail[:end] 165 tail = tail[end:] 166 head = ''.join(chunks) 167 lines = head.split('\r\n') 168 if macho.mode == 'server' and lines[0].startswith('TUNNEL ') and lines[0].endswith(' TCPTUNNEL/1.0') and self.tunnelinfo is None: 169 proto, info, version = lines[0].split(' ') 170 self.tunnelinfo = blue.marshal.Load(binascii.a2b_hex(info)) 171 setattr(self.ep, '_tunnelinfo', self.tunnelinfo) 172 lines.pop(0) 173 else: 174 self.tunnelinfo = getattr(self.ep, '_tunnelinfo', None) 175 headers = HeaderDict() 176 for line in lines[1:]: 177 try: 178 key, val = line.split(':', 1) 179 except ValueError: 180 continue 181 182 headers[key] = val.strip() 183 184 cl = typeutils.int_eval(headers.get('Content-Length', 0), 0) 185 if cl: 186 got = len(tail) 187 chunks = [tail] 188 while got < cl: 189 missing = cl - got 190 ask = min(missing, 4096) 191 get = self.ep.Read(ask) 192 got += len(get) 193 chunks.append(get) 194 195 content = ''.join(chunks) 196 tail = content[cl:] 197 content = content[:cl] 198 else: 199 content = '' 200 self.host = headers.get('host', '') 201 self.node_id = self.httpService.machoNet.GetNodeID() 202 self.macho_mode = macho.mode 203 if self.tunnelinfo: 204 if 'requesterIP' in self.tunnelinfo: 205 if 'x-forwarded-for' in headers: 206 headers['x-forwarded-for'] = '%s, %s' % (headers['x-forwarded-for'], self.tunnelinfo['requesterIP']) 207 else: 208 headers['x-forwarded-for'] = self.tunnelinfo['requesterIP'] 209 self.client_ip = self.tunnelinfo['requesterIP'] 210 if 'nodeID' in self.tunnelinfo: 211 self.via_node_id = self.tunnelinfo['nodeID'] 212 else: 213 try: 214 self.via_node_id = self.node_id 215 self.client_ip = self.ep.socket.getpeername()[0] 216 except Exception: 217 pass 218 219 self.buff = tail 220 return (lines[0], 221 headers, 222 head, 223 content) 224 225 def ParseHeader(self): 226 request, self.header, head, body = self.ReadRequest() 227 self.raw = ''.join((head, body)) 228 line = request 229 sp = line.find(' ') 230 self.method = line[:sp] 231 self.proto = line[-9:].strip() 232 if self.proto not in ('HTTP/1.0', 'HTTP/1.1'): 233 raise UnknownProtocol(self.proto) 234 if self.method not in ('GET', 'POST', 'HEAD', 'OPTIONS'): 235 raise UnknownVerb(self.method) 236 self.full_path = line[sp + 1:-9] 237 self.path = self.full_path 238 argidx = self.path.find('?') 239 if argidx != -1: 240 self.args = self.path[argidx + 1:] 241 self.path = self.path[:argidx] 242 htmlwriter.SplitArgs(self.args, self.query) 243 else: 244 import urllib 245 self.path = urllib.unquote_plus(self.path) 246 if self.path.startswith('/'): 247 self.paths = self.path[1:].split('/') 248 else: 249 self.paths = [] 250 if 'Cookie' in self.header: 251 cookies = self.header['Cookie'] 252 for cookie in cookies.split(';'): 253 kv = cookie.split('=', 1) 254 if len(kv) == 2: 255 self.cookie[kv[0].strip()] = kv[1].strip() 256 257 if self.httpService.httpLog: 258 if self.httpService.logChannel.IsOpen(1): 259 for y in utillib.strx(self.buff[:-4]).split('\n'): 260 for l in LineWrap(y.replace('\r', ''), maxlines=0, maxlen=80, pfx=''): 261 self.httpService.LogMethodCall('HTTP INPUT: ', l) 262 263 if 'Expect' in self.header and self.header['Expect'].lower().strip().find('100-continue') >= 0: 264 self.ep.Write('HTTP/1.1 100 Continue\r\n\r\n') 265 if self.httpService.httpLog: 266 self.httpService.LogMethodCall('HTTP OUTPUT: HTTP/1.1 100 Continue') 267 self.httpService.LogMethodCall('HTTP OUTPUT: ') 268 if self.method == 'POST': 269 form = body 270 if form: 271 if self.httpService.httpLog: 272 if self.httpService.logChannel.IsOpen(1): 273 for y in utillib.strx(form).split('\n'): 274 for l in LineWrap(y.replace('\r', ''), maxlines=0, maxlen=80, pfx=''): 275 self.httpService.LogMethodCall('HTTP INPUT: ', l) 276 277 if self.header['Content-Type'].find('multipart/form-data') == -1: 278 htmlwriter.SplitArgs(form, self.form) 279 else: 280 hd, body = head, body 281 req, hd = hd.split('\r\n', 1) 282 headers = hd.split('\r\n') 283 headers = [ header.split(': ', 1) for header in headers ] 284 headers = [ header for header in headers if len(header) > 1 ] 285 headers = dict([ (k.lower(), v) for k, v in headers ]) 286 method = req.split(' ', 1)[0] 287 self.formfields = cgi.FieldStorage(StringIO.StringIO(body), headers=headers, strict_parsing=1, environ={'REQUEST_METHOD': method}) 288 htmlwriter.SplitMIMEArgs(self.formfields, self.form) 289 if 'action' in self.form and 'action' not in self.query: 290 action = self.form['action'] 291 if action != 'ActionTakenFromQuery': 292 self.query['action'] = action 293 294 def SaveFiles(self, path): 295 ret = [] 296 if self.formfields: 297 for field in self.formfields.list: 298 fileitem = self.formfields[field.name] 299 if fileitem.filename: 300 if path[:1] != '\\': 301 path = '%s\\' % path 302 sourceFileName = fileitem.filename 303 fileName = os.path.basename(sourceFileName) 304 exactFileName = path + fileName 305 ret.append({'size': len(field.value), 306 'filename': fileName, 307 'sourceFileName': sourceFileName, 308 'path': path, 309 'exactFileName': exactFileName}) 310 fout = file(exactFileName, 'wb') 311 while 1: 312 chunk = fileitem.file.read(100000) 313 if not chunk: 314 break 315 fout.write(chunk) 316 317 fout.close() 318 319 return ret 320 321 def GetFiles(self): 322 ret = {} 323 if self.formfields: 324 for field in self.formfields.list: 325 fileitem = self.formfields[field.name] 326 if fileitem.filename: 327 sourceFileName = fileitem.filename 328 fileName = os.path.basename(sourceFileName) 329 ret[fileitem.filename] = {'size': len(field.value), 330 'filename': fileName, 331 'fileData': fileitem.file.read(len(field.value))} 332 333 return ret 334 335 def DumpRequest(self, withRaw = False): 336 print('************** R E Q U E S T ******************') 337 print('my url:', self.path) 338 print('my paths:', self.paths) 339 print('my method:', self.method) 340 print('my proto:', self.proto) 341 print('my args', self.args) 342 print('my byte count', self.totalBytes) 343 for q in self.query.iteritems(): 344 print('query var %s=%s' % q) 345 346 for f in self.form.iteritems(): 347 print('form var %s=%s' % f) 348 349 print('server varirables:') 350 for i in self.header.iteritems(): 351 print(i[0], ':', i[1]) 352 353 if withRaw: 354 print('------------------- R A W ---------------------') 355 print(self.raw) 356 print('***********************************************') 357 358 def DumpRequestToList(self): 359 res = ['request.uid: %s' % self.uid, 360 '----------', 361 'my url: %s' % self.path, 362 'my paths: %s' % self.paths, 363 'my method: %s' % self.method, 364 'my proto: %s' % self.proto, 365 'my args %s' % self.args, 366 '----------'] 367 for q in self.query.iteritems(): 368 res.append('query var %s=%s' % q) 369 370 res.append('----------') 371 res.append('SERVER VARIABLES:') 372 for i in self.header.iteritems(): 373 res.append('%s:%s' % (i[0], i[1])) 374 375 res.append('request.node_id: %s' % self.node_id) 376 res.append('request.via_node_id: %s' % self.via_node_id) 377 res.append('request.client_ip: %s' % self.client_ip) 378 return res 379 380 def Authorization(self): 381 if 'Authorization' in self.header: 382 try: 383 uspa = self.header.pop('Authorization') 384 i = uspa.rfind(' ') 385 uspa = uspa[i + 1:] 386 import base64 387 uspa = base64.decodestring(uspa) 388 try: 389 uspa = unicode(uspa, encoding='utf-8') 390 except Exception as ex: 391 try: 392 uspa = unicode(uspa, encoding='iso-8859-1') 393 except Exception as ex: 394 pass 395 396 i = uspa.find(':') 397 username = uspa[:i] 398 password = uspa[i + 1:] 399 return (username, PasswordString(password)) 400 except: 401 pass 402 403 404 def GetSession(parent, request, response, sessionsBySID, sessionsByFlatkaka): 405 uspa = request.Authorization() 406 if 'flatkaka' in request.cookie: 407 flatkaka = request.cookie['flatkaka'] 408 if flatkaka in sessionsByFlatkaka: 409 sess = sessionsByFlatkaka[flatkaka] 410 if macho.mode == 'client': 411 return sess 412 if uspa is not None: 413 u = sess.esps.contents['username'] 414 p = sess.esps.contents['password'] 415 if uspa[0] != u or uspa[1] != p: 416 svclog.warning("User %s is trying to hijack %s's session, with sessID=%d", uspa[0], u, sess.sid) 417 else: 418 return sess 419 sess = None 420 success = False 421 username = None 422 password = None 423 reason = 'Internal Server Error' 424 statusCode = '500 Internal Server Error' 425 if macho.mode == 'client': 426 sess = basesession.CreateSession(None, const.session.SESSION_TYPE_ESP, None) 427 sess.esps = ESPSession(parent, sess.sid) 428 success = True 429 else: 430 usernameAndPassword = uspa 431 reason = 'Access denied' 432 statusCode = '401 Unauthorized' 433 if usernameAndPassword is not None: 434 username = usernameAndPassword[0] 435 password = usernameAndPassword[1] 436 for s in sessionsBySID.itervalues(): 437 if hasattr(s, 'esps') and s.esps.contents['username'] == username: 438 if s.userid and s.esps.contents['password'] == password: 439 return s 440 break 441 442 if macho.mode == 'server' and ('authentication' not in sm.services or sm.services['authentication'].state != SERVICE_RUNNING): 443 blue.pyos.synchro.SleepWallclock(3000) 444 raise UserError('AutClusterStarting') 445 try: 446 if sm.services['http'].session.ConnectToProxyServerService('machoNet').CheckACL(request.ep.address, espCheck=True): 447 blue.pyos.synchro.SleepWallclock(3000) 448 raise UserError('AutClusterStarting') 449 except machoNetExceptions.UnMachoDestination: 450 blue.pyos.synchro.SleepWallclock(3000) 451 raise UserError('AutClusterStarting') 452 453 sessionID = basesession.GetNewSid() 454 sess = basesession.CreateSession(sessionID, const.session.SESSION_TYPE_ESP) 455 sess.esps = ESPSession(parent, sess.sid) 456 auth = basesession.GetServiceSession('httpService').ConnectToAnyService('authentication') 457 try: 458 sessstuff = None 459 try: 460 sessstuff, _, _ = auth.Login(sessionID, username, password, None, const.userConnectTypeServerPages, request.ep.address) 461 sessstuff['role'] |= sess.role 462 except UserError as e: 463 if e.msg != 'CharacterInDifferentRegion': 464 raise 465 sys.exc_clear() 466 467 if sessstuff: 468 if 'userid' in sessstuff: 469 for each in basesession.FindSessions('userid', [sessstuff['userid']]): 470 each.LogSessionHistory('Usurped by user %s via HTTP using local authentication' % username) 471 basesession.CloseSession(each) 472 473 sess.SetAttributes(sessstuff) 474 sess.LogSessionHistory('Authenticated user %s via HTTP using local authentication' % username) 475 success = True 476 except machoNetExceptions.UnMachoDestination: 477 reason = 'The proxy server was unable to connect to any Sol Server Node to handle your authentication request.' 478 statusCode = '500 No Sol Server available' 479 sys.exc_clear() 480 except UserError as e: 481 if e.msg not in ('LoginAuthFailed', 'LegacyLoginDisabled'): 482 raise 483 sys.exc_clear() 484 485 if not success: 486 sess.LogSessionHistory('Session closed due to local authentication failure') 487 basesession.CloseSession(sess) 488 if success: 489 sessID = sess.sid 490 while sessID in sessionsBySID: 491 svclog.warning('Session %d already exits, adding 1 to it', sessID) 492 sessID += 1 493 494 sessionsBySID[sessID] = sess 495 sessionsByFlatkaka[sess.esps.GetFlatkaka()] = sess 496 parent.OnSessionBegin(sessID) 497 sess.cacheList = [] 498 sess.requestCount = 0 499 sess.esps.contents['timeoutTimer'] = None 500 if macho.mode != 'client': 501 sess.esps.contents['username'] = username 502 sess.esps.contents['password'] = password 503 if macho.mode in ('proxy', 'server'): 504 sm.ChainEvent('ProcessESPSessionCreated', sessionID=sessionID, ipaddress=request.ep.address) 505 return sess 506 else: 507 response.Clear() 508 response.status = statusCode 509 response.Write(reason) 510 response.authenticate = 1 511 response.Flush() 512 return 513 514 515 class Response(): 516 517 def __init__(self, httpService, ep): 518 self.httpService = httpService 519 self.ep = ep 520 self.streamMode = None 521 self.buff = cStringIO.StringIO() 522 self.cookie = {} 523 self.contentType = 'text/HTML; charset=UTF-8' 524 self.status = '200 OK' 525 self.header = {} 526 self.authenticate = 0 527 self.done = 0 528 self.is_python = False 529 530 def AddHeader(self, name, value): 531 self.header[name] = value 532 533 def AppendToLog(self, buff): 534 raise RuntimeError('AppendToLog not implemented just yet', buff) 535 536 def WriteBinary(self, buff): 537 if self.streamMode == 't': 538 raise RuntimeError('You cannot call WriteBinary() after calling Write()') 539 self.streamMode = 'b' 540 self.buff.write(buffer(buff)) 541 542 def Write(self, buff): 543 if self.streamMode == 'b': 544 raise RuntimeError('You cannot call Write() after calling WriteBinary()') 545 self.streamMode = 't' 546 self.buff.write(buff.encode('utf-8')) 547 self.buff.write('\r\n') 548 549 def Clear(self): 550 self.buff.close() 551 self.streamMode = None 552 self.buff = cStringIO.StringIO() 553 554 def End(self): 555 raise RuntimeError('End not implemented just yet') 556 557 def Flush(self): 558 if self.done: 559 return 560 self.done = 1 561 self.buff.seek(0, 2) 562 self.header['Content-Length'] = self.buff.tell() 563 if 'Content-Type' not in self.header: 564 self.header['Content-Type'] = self.contentType 565 self.header['Server'] = 'CCP-SP/%s' % boot.version 566 if 'Keep-Alive' not in self.header: 567 self.header['Keep-Alive'] = 'timeout=15, max=98' 568 if 'Connection' not in self.header: 569 self.header['Connection'] = 'Keep-Alive' 570 if self.authenticate: 571 self.header['WWW-Authenticate'] = 'Basic realm="CCP SERVER PAGES"' 572 for k, v in self.httpService.GetStaticHeader().iteritems(): 573 if k not in self.header: 574 self.header[k] = v 575 576 if self.cookie: 577 s = ''.join([ '%s=%s; ' % (k, v) for k, v in self.cookie.iteritems() ]) 578 self.header['Set-Cookie'] = s + 'path=/' 579 s = cStringIO.StringIO() 580 s.write('HTTP/1.1 ') 581 s.write(str(self.status)) 582 s.write('\r\n') 583 for k, v in self.header.iteritems(): 584 s.write(k) 585 s.write(': ') 586 s.write(str(v)) 587 s.write('\r\n') 588 589 s.write('\r\n') 590 e = self.buff.getvalue() 591 s.write(e) 592 out = s.getvalue() 593 if self.httpService.httpLog: 594 if self.httpService.logChannel.IsOpen(1): 595 for y in out.split('\n'): 596 for l in LineWrap(y.replace('\r', ''), maxlines=0, maxlen=80, pfx=''): 597 self.httpService.LogMethodCall('HTTP OUTPUT: ', utillib.strx(l)) 598 599 self.ep.Write(out) 600 601 def Redirect(self, url, args = None, **kwargs): 602 self.status = '302 Object Moved' 603 if kwargs: 604 if args and isinstance(args, dict): 605 args.update(kwargs) 606 else: 607 args = kwargs 608 if args: 609 url = '%s?%s' % (url, urllib.urlencode(args)) 610 self.header['Location'] = url.encode('UTF-8') 611 self.Flush() 612 613 def SendNotModified(self, path): 614 self.header['Location'] = path.encode('UTF-8') 615 self.status = '304 Not Modified' 616 self.Flush() 617 618 def SendNotImplemented(self): 619 self.status = '501 Not Implemented' 620 self.Flush() 621 622 623 class HeadResponse(Response): 624 625 def Write(self, buff): 626 pass 627 628 WriteBinary = Write 629 630 631 class ESPSession(): 632 633 def __init__(self, owner, sid): 634 self.codePage = 0 635 self.contents = {} 636 self.LCID = 0 637 self.sessionID = sid 638 self.timeout = 20 639 self.authenticated = 0 640 self.username = '' 641 self.password = '' 642 self.owner = owner 643 self.flatkokudeig = blue.os.GetWallclockTimeNow() 644 self.remappings = {} 645 646 def GetFlatkaka(self): 647 return binascii.b2a_hex(Crypto.CryptoHash(self.flatkokudeig, id(self.owner), self.sessionID, sm.services['machoNet'].GetNodeID())) 648 649 def Abandon(self): 650 self.owner.OnSessionEnd(self.sessionID) 651 self.owner = None 652 653 654 class ConnectionService(Service): 655 __startupdependencies__ = ['dataconfig', 'machoNet', 'counter'] 656 __dependencies__ = ['machoNet'] 657 __exportedcalls__ = {'GetCacheStatus': [ROLE_SERVICE | ROLE_ADMIN], 658 'SetCacheStatus': [ROLE_SERVICE | ROLE_ADMIN], 659 'GetCacheSkipList': [ROLE_SERVICE | ROLE_ADMIN], 660 'AddToCacheSkipList': [ROLE_SERVICE], 661 'GetStaticHeader': [ROLE_SERVICE]} 662 __configvalues__ = {'httpLog': 0} 663 __guid__ = 'svc.http' 664 __notifyevents__ = ['OnSessionEnd'] 665 __counters__ = {'openConnectionsInHttpService': 'normal'} 666 667 def __init__(self): 668 CoreService.__init__(self) 669 self.quit = 0 670 self.sessionsBySID = {} 671 self.sessionsByFlatkaka = {} 672 self.connections = [] 673 self.cacheSkipList = [] 674 self.caching = 0 675 self.acceptingHTTP = 0 676 self.codeCache = {} 677 self.threads = 0 678 self.TimoutOutIntervalInMinutes = 5 679 self.staticHeader = None 680 self.socket = None 681 self.state = SERVICE_STOPPED 682 683 def GetHtmlStateDetails(self, k, v, detailed): 684 return None 685 686 def OnSessionBegin(self, sessionID): 687 if sessionID in self.sessionsBySID: 688 sess = self.sessionsBySID[sessionID] 689 sess.esps = ESPSession(self, sessionID) 690 691 def OnSessionEnd(self, sessionID): 692 if sessionID in self.sessionsBySID: 693 sess = self.sessionsBySID[sessionID] 694 del self.sessionsBySID[sessionID] 695 if getattr(sess, 'esps', None): 696 kaka = sess.esps.GetFlatkaka() 697 if kaka in self.sessionsByFlatkaka: 698 del self.sessionsByFlatkaka[kaka] 699 sess.LogSessionHistory('Session closed during OnSessionEnd') 700 basesession.CloseSession(sess) 701 702 def Init(self): 703 if boot.role == 'client' and blue.pyos.packaged: 704 raise RuntimeError("Http service: can't run") 705 self.quit = 0 706 self.sessionsBySID = {} 707 self.sessionsByFlatkaka = {} 708 self.connections = [] 709 self.cacheSkipList = ['.py', '.htc'] 710 self.caching = 0 711 self.acceptingHTTP = int(prefs.GetValue('http', ['1', '0'][macho.mode == 'client'])) 712 self.codeCache = {} 713 self.threads = 0 714 self.TimoutOutIntervalInMinutes = 5 715 self.staticHeader = None 716 717 def Run(self, memStream = None): 718 self.Init() 719 if self.acceptingHTTP == 0: 720 svclog.info('Http server not running as no http=1 in prefs.ini') 721 return 722 if prefs.GetValue('httpServerMode', 'ccp').lower() != 'ccp': 723 svclog.info('Http server not running as httpServerMode!=ccp in prefs.ini') 724 return 725 self.socket = self.machoNet.GetTransport('tcp:raw:http') 726 svclog.info('Http server running on address %s', self.socket.address) 727 for i in xrange(5): 728 self.StartThread() 729 730 def StartThread(self): 731 self.threads += 1 732 uthread.pool('http::AcceptThread', self.AcceptThread) 733 734 def Stop(self, memStream = None): 735 self.quit = 1 736 self.state = SERVICE_STOP_PENDING 737 738 def AcceptThread(self): 739 try: 740 try: 741 svclog.debug('Accepting http over transport: %s', self.socket) 742 ep = self.socket.Accept() 743 except GPSExceptions.GPSTransportClosed: 744 if self.quit == 1: 745 svclog.debug('Accept returned because the service is stopping') 746 raise 747 748 svclog.debug('Accepted a connection: %s', ep) 749 if not self.quit: 750 self.StartThread() 751 self.HandleConnection(ep) 752 finally: 753 if hasattr(self, 'threads'): 754 self.threads -= 1 755 if self.threads == 0: 756 self.state = SERVICE_STOPPED 757 758 def GetCacheStatus(self): 759 return self.caching 760 761 def SetCacheStatus(self, status): 762 self.caching = status 763 764 def GetCacheSkipList(self): 765 result = [] 766 for s in self.cacheSkipList: 767 result.append([s]) 768 769 return result 770 771 def AddToCacheSkipList(self, value): 772 self.cacheSkipList.append(value) 773 774 def GetStaticHeader(self): 775 uthread.Lock('http.GetStaticHeader') 776 try: 777 if self.staticHeader is None: 778 self.staticHeader = {'CCP-codename': boot.codename, 779 'CCP-version': boot.version, 780 'CCP-build': boot.build, 781 'CCP-sync': boot.sync, 782 'CCP-clustermode': prefs.GetValue('clusterMode', 'n/a')} 783 if boot.role == 'server': 784 product = sm.GetService('cache').Setting('zsystem', 'Product') 785 ebsVersion = sm.GetService('DB2').CallProc('zsystem.Versions_Select', product)[0].version 786 self.staticHeader['CCP-product'] = product 787 self.staticHeader['CCP-EBS'] = ebsVersion 788 if boot.role == 'proxy': 789 machoNet = sm.GetService('machoNet') 790 onlineCountEve = machoNet.GetClusterSessionCounts('EVE:Online')[0] 791 onlineCountDust = machoNet.GetClusterSessionCounts('DUST:Online')[0] 792 acl = machoNet.CheckACL(None, espCheck=True) 793 self.staticHeader['CCP-onlineCount'] = onlineCountEve + onlineCountDust 794 self.staticHeader['CCP-onlineCountEve'] = onlineCountEve 795 self.staticHeader['CCP-onlineCountDust'] = onlineCountDust 796 self.staticHeader['CCP-ACL'] = acl[1] if acl else None 797 self.staticHeader['CCP-VIP'] = machoNet.vipMode 798 except Exception: 799 self.staticHeader = {'BADHEADER': ''} 800 log.LogException() 801 finally: 802 uthread.UnLock('http.GetStaticHeader') 803 804 return self.staticHeader 805 806 def RemoveFromCacheSkipList(self, value): 807 pass 808 809 def HandleConnection(self, ep): 810 if not hasattr(self, 'caching'): 811 ClockThis('HTTP::Handle::Init', self.Init) 812 requestCount = 0 813 sess = None 814 self.openConnectionsInHttpService.Add() 815 try: 816 while 1: 817 sys.exc_clear() 818 request = Request(self, ep) 819 if request.method != 'HEAD': 820 response = Response(self, ep) 821 else: 822 response = HeadResponse(self, ep) 823 errfile = None 824 lastRequest = request.path 825 try: 826 ClockThis('HTTP::Handle::request.ParseHeader', request.ParseHeader) 827 if request.method == 'OPTION': 828 tmp = request.DumpRequestToList() 829 svclog.error('OPTION REQUEST, sorry not really an error') 830 svclog.error('The client %s made this request', ep.address) 831 for s in tmp: 832 svclog.error(s) 833 834 response.SendNotImplemented() 835 continue 836 requestCount += 1 837 sess = self.GetSession(request, response) 838 if sess: 839 sess.requestCount += 1 840 else: 841 continue 842 if self.HandleCaching(ep, request, response): 843 continue 844 response.cookie['flatkaka'] = sess.esps.GetFlatkaka() 845 filename, files = self.GetFileFromRequest(request) 846 if not files: 847 errfile = filename 848 raise IOError('file %s not found in www roots' % filename) 849 try: 850 self.HandleRequestFile(request, response, filename, files) 851 finally: 852 for _, f in files: 853 f.Close() 854 855 except GPSExceptions.GPSTransportClosed: 856 svclog.debug('closed retrieving [%s], before that [%s].', request.path, lastRequest) 857 svclog.debug('Total requests served with this connection: %d', requestCount) 858 break 859 except UnknownProtocol as e: 860 httplog.warning('%s sent Unknown HTTP protocol "%s" - method=%s, host=%s, path=%s, userid=%s, nodeID=%s/%s via=%s', request.client_ip, request.proto, request.method, request.host, request.full_path, getattr(request.session, 'userid', None), request.node_id, request.macho_mode, request.via_node_id) 861 break 862 except UnknownVerb as e: 863 httplog.warning('%s sent Unknown HTTP method "%s" - host=%s, path=%s, userid=%s, nodeID=%s/%s via=%s', request.client_ip, request.method, request.host, request.full_path, getattr(request.session, 'userid', None), request.node_id, request.macho_mode, request.via_node_id) 864 break 865 except Exception as ex: 866 self.HandleException(sess, request, response, ex, errfile) 867 868 try: 869 response.Flush() 870 if response.is_python: 871 httplog.info('[%s] Response %s in %s sec. length=%s, type=%s, stream=%s', request.uid, response.status, time.time() - request.start_time, response.header.get('Content-Length', -1), response.header.get('Content-Type', ''), response.streamMode) 872 if request.proto == 'HTTP/1.0': 873 break 874 except GPSExceptions.GPSTransportClosed: 875 sys.exc_clear() 876 svclog.warning('Trying to send response for [%s] but the connection was closed, prev: %s', request.path, lastRequest) 877 svclog.warning('Total requests served with this connection: %d', requestCount) 878 if response.is_python: 879 httplog.info('[%s] Failed to send responce - Socket closed after %s sec. length=%s, type=%s, stream=%s', request.uid, time.time() - request.start_time, response.header.get('Content-Length', -1), response.header.get('Content-Type', ''), response.streamMode) 880 break 881 882 finally: 883 if not getattr(ep.socket, 'isFake', False): 884 ep.Close() 885 self.openConnectionsInHttpService.Dec() 886 if sess: 887 if 'timeoutTimer' in sess.esps.contents: 888 if sess.esps.contents['timeoutTimer'] is None: 889 sess.esps.contents['timeoutTimer'] = 1 890 uthread.new(self.CheckSessionTimeout, sess, sess.requestCount) 891 892 def GetSession(self, request, response): 893 try: 894 if not request.session.userid: 895 raise AttributeError 896 return request.session 897 except AttributeError: 898 sess = ClockThis('HTTP::Handle::GetSession', GetSession, self, request, response, self.sessionsBySID, self.sessionsByFlatkaka) 899 if sess: 900 request.session = sess 901 return sess 902 903 def HandleCaching(self, ep, request, response): 904 if not self.caching: 905 return False 906 else: 907 r = request.path.rfind('.') 908 if r == -1: 909 return False 910 fileType = request.path[r:] 911 for each in self.cacheSkipList: 912 if fileType.startswith(each.lower()): 913 return False 914 915 sess = request.session 916 if request.path in sess.cacheList: 917 response.SendNotModified(request.path) 918 response.cookie['flatkaka'] = sess.esps.GetFlatkaka() 919 return True 920 sess.cacheList.append(request.path) 921 svclog.debug('adding %s to cacheList', request.path) 922 return False 923 924 def GetFileFromRequest(self, request): 925 sess = request.session 926 if request.path in sess.esps.remappings: 927 filename = sess.esps.remappings[request.path] 928 svclog.debug('Remapped %s as %s', request.path, filename) 929 else: 930 if request.path[:7] == '/cache/': 931 filename = 'cache:' + request.path[6:] 932 else: 933 filename = 'wwwroot:/' + request.path 934 if filename[-1:] == '/': 935 filename = '%sdefault.py' % filename 936 if not sess.role & ROLE_PROGRAMMER: 937 refuse = False 938 if '..' in request.path: 939 svclog.error("Refusing to give %s to %s because he's trying to use .. in the filename to hax0r us :(", filename, sess.userid) 940 refuse = True 941 else: 942 extidx = filename.rfind('.') 943 if extidx == -1: 944 svclog.error("Refusing to give %s to %s because he's trying to get a file that has no extension and hax0r us :(", filename, sess.userid) 945 refuse = True 946 else: 947 ext = filename[extidx + 1:] 948 if ext.lower() not in self._GetDownloadableFileExtensions(): 949 svclog.error("Refusing to give %s to %s because he's trying to get a file that has an extension that only programmers may acquire :(", filename, sess.userid) 950 refuse = True 951 if refuse: 952 raise AuthError('Only programmers may download arbitrary stuff from servers') 953 ret = [] 954 fullPath = blue.paths.ResolvePath(filename) 955 if os.path.exists(fullPath): 956 resFile = blue.ResFile() 957 resFile.OpenAlways(fullPath, 1) 958 ret = [(fullPath, resFile)] 959 return (filename, ret) 960 961 def HandleRequestFile(self, request, response, filename, files): 962 if filename[-3:] == '.py': 963 response.is_python = True 964 self.HandlePython(request, response, filename, files) 965 else: 966 resFile = files[-1][1] 967 response.contentType = httputils.get_mimetype(filename) 968 response.header['Cache-Control'] = 'max-age=31536000000, public' 969 if request.method == 'HEAD': 970 svclog.debug('Got a HEAD request, not returning body...') 971 else: 972 s = resFile.size 973 if s > 1048576: 974 data = uthread.CallOnThread(resFile.Read) 975 else: 976 data = resFile.Read() 977 response.WriteBinary(data) 978 979 def HandlePython(self, request, response, filename, files): 980 oldctxt = bluepy.PushTimer('HTTP::Handle::Pages') 981 try: 982 response.header['Expires'] = '0' 983 response.header['Cache-Control'] = 'private, no-cache' 984 if request.method == 'HEAD': 985 svclog.debug('Got a HEAD request, not executing script...') 986 else: 987 request.uid = uuid.uuid4() 988 httplog.info('%s userID=%s %s %s %s [%s] nodeID=%s/%s via=%s', request.client_ip, getattr(request.session, 'userid', None), request.method, request.full_path, request.host, request.uid, request.node_id, request.macho_mode, request.via_node_id) 989 if request.method in ('POST', 'PUT'): 990 httplog.info('[%s] %s data %r', request.uid, request.method, request.form) 991 modified = max([ os.stat(fullpath).st_mtime for fullpath, f in files ]) 992 if filename not in self.codeCache or modified > self.codeCache[filename][0]: 993 with bluepy.Timer('HTTP::Handle::ExecFile::' + filename): 994 import __builtin__ 995 glob = {'__builtins__': __builtin__} 996 for fn, f in files: 997 data = f.read().replace('\r\n', '\n') 998 f.close() 999 code = compile(data, fn, 'exec', 0, True) 1000 exec code in glob 1001 1002 self.codeCache[filename] = (modified, glob) 1003 else: 1004 glob = self.codeCache[filename][1] 1005 sess = request.session 1006 masque = sess.Masquerade() 1007 if not session.userid and macho.mode != 'client': 1008 raise RuntimeError('\n **********************************************************************\n * SESSION IS BROKED, HAS NO USERID. RESTART YOUR SERVER PAGE BROWSER *\n **********************************************************************\n ') 1009 try: 1010 ClockThis('HTTP::Handle::Pages::' + request.path, glob['Execute'], request, response, sess) 1011 finally: 1012 masque.UnMask() 1013 1014 finally: 1015 bluepy.PopTimer(oldctxt) 1016 1017 def _TrackPage(self, title, session): 1018 trackID = 1 if prefs.clusterMode == 'LOCAL' else prefs.GetValue('webTrackID', -1) 1019 trackUrl = prefs.GetValue('trackUrl', 'http://piwik/') 1020 if not trackUrl.endswith('/'): 1021 trackUrl += '/' 1022 if trackID > -1: 1023 return '\n <!-- Piwik -->\n <script type="text/javascript">\n var pkBaseURL = (("https:" == document.location.protocol) ? "%(trackHttps)s" : "%(trackHttp)s");\n document.write(unescape("%%3Cscript src=\'" + pkBaseURL + "piwik.js\' type=\'text/javascript\'%%3E%%3C/script%%3E"));\n </script><script type="text/javascript">\n try {\n var user = { \'userID\' : %(userid)s };\n var piwikTracker = Piwik.getTracker(pkBaseURL + "piwik.php", %(trackID)d);\n piwikTracker.setCustomData(user);\n piwikTracker.setDocumentTitle(\'%(documentTitle)s\');\n piwikTracker.trackPageView();\n piwikTracker.enableLinkTracking();\n } catch(err) {}\n </script><noscript><p><img src="%(trackHttp)spiwik.php?idsite=%(trackID)d" style="border:0" alt="" /></p></noscript>\n <!-- End Piwik Tracking Tag -->\n ' % {'userid': session.userid if session and session.userid is not None else 'Not Authorized', 1024 'documentTitle': 'title', 1025 'trackID': trackID, 1026 'trackHttp': trackUrl, 1027 'trackHttps': trackUrl.replace('http:', 'https:')} 1028 1029 def HandleException(self, sess, request, response, error, errfile): 1030 req_session = getattr(request, 'session', None) 1031 if isinstance(error, UserError): 1032 log.LogException(toAlertSvc=0, toMsgWindow=0, severity=log.LGWARN) 1033 if error.msg == 'Unknown protocol' or error.msg == 'Unknown command': 1034 response.Clear() 1035 response.Write('<html><head>') 1036 response.Write('\t<title>Internal Server Error</title>') 1037 response.Write('\t<link rel="stylesheet" href="/lib/error.css"/>') 1038 response.status = 501 1039 response.Write('</head><body>%s</body><img src="/img/header_error.jpg">' % str(error)) 1040 elif error.msg == 'AutClusterStarting': 1041 response.Clear() 1042 response.Write('<html><head>') 1043 response.Write('\t<title>Authentication Failure</title>') 1044 response.Write('\t<link rel="stylesheet" href="/lib/std.css"/>') 1045 response.status = '401 Unauthorized' 1046 response.Write('</head><body><h1>Cluster Startup in Progress</h1>The cluster is not yet accepting incoming connections.</body>') 1047 else: 1048 response.Clear() 1049 response.Write('<html><head>') 1050 response.Write('\t<title>Exception - UserError</title>') 1051 response.Write('\t<link rel="stylesheet" href="/lib/error.css"/>') 1052 response.status = '200 OK' 1053 response.Write('</head>\n<body><img src="/img/header_error.jpg"><h1>UserError</h1>') 1054 if hasattr(__builtins__, 'cfg'): 1055 response.Write('<font size=4>%s</font><br><br><br>' % cfg.Format(error.msg, error.dict)) 1056 else: 1057 response.Write('<font size=4>%s</font><br><br><br>' % utillib.strx((error.msg, error.dict))) 1058 response.Write('<pre>') 1059 response.Write(''.join(traceback.format_exc())) 1060 response.Write('</pre></body></html>') 1061 elif isinstance(error, machoNetExceptions.WrongMachoNode): 1062 if not error.payload: 1063 raise error 1064 targetNodeID = error.payload 1065 notFoundMacho = True 1066 current_host, current_port = ('', '') 1067 try: 1068 func = lambda x: (True if x.lower().startswith('host') else False) 1069 host = filter(func, request.raw.split('\r\n'))[0].split(':') 1070 current_host, current_port = host[1].strip(), host[2].strip() 1071 except Exception: 1072 log.LogException() 1073 1074 if isinstance(targetNodeID, int): 1075 if sm.IsServiceRunning('tcpRawProxyService'): 1076 tcpproxy = sm.services['tcpRawProxyService'] 1077 else: 1078 proxyID = sm.services['machoNet'].GetConnectedProxyNodes()[0] 1079 tcpproxy = sm.StartService('debug').session.ConnectToRemoteService('tcpRawProxyService', proxyID) 1080 host, ports = tcpproxy.GetESPTunnelingAddressByNodeID() 1081 port = ports.get(targetNodeID) 1082 if str(host).lower() != str(current_host).lower() or port != int(current_port): 1083 notFoundMacho = False 1084 protocol = 'https' if iocp.UsingHTTPS() else 'http' 1085 url = '%s://%s:%s%s' % (protocol, 1086 host, 1087 port, 1088 request.FullPath()) 1089 response.Redirect(url) 1090 if notFoundMacho: 1091 response.Clear() 1092 response.Write('<html><head>') 1093 response.Write('\t<title>Failed macho redirect</title>') 1094 response.Write('\t<link rel="stylesheet" href="/lib/error.css">') 1095 response.status = '404 Not Found' 1096 response.Write('</head>\n<body><img src="/img/header_error.jpg">') 1097 response.Write("<h1>Don't know how to redirect to node %s</h1>" % (targetNodeID,)) 1098 response.Write('<br>file %r %r' % (errfile, request.path)) 1099 response.Write(' ' * 512) 1100 response.Write('%s</body></html>' % self._TrackPage('404 Not Found', req_session)) 1101 elif isinstance(error, IOError): 1102 response.Clear() 1103 response.Write('<html><head>') 1104 response.Write('\t<title>404 Not Found</title>') 1105 response.Write('\t<link rel="stylesheet" href="/lib/error.css">') 1106 response.status = '404 Not Found' 1107 response.Write('</head>\n<body><img src="/img/header_error.jpg"><h1>404 Not Found</h1>') 1108 response.Write('<br>file %r %r' % (errfile, request.path)) 1109 response.Write('<br>error.errno=%s' % error.errno) 1110 response.Write('<br>error.filename=%s' % error.filename) 1111 response.Write('<br>error.strerror=%s' % error.strerror) 1112 response.Write('<br>error.args=%r' % error.args) 1113 response.Write('<br>error=%r' % error) 1114 response.Write('<br>request.uid=%s' % request.uid) 1115 response.Write(' ' * 512) 1116 response.Write('%s</body></html>' % self._TrackPage('404 Not Found', req_session)) 1117 elif isinstance(error, AuthError): 1118 response.Clear() 1119 response.Write('<html><head>') 1120 response.Write('\t<title>401 Unauthorized</title>') 1121 response.Write('\t<link rel="stylesheet" href="/lib/std.css"/>') 1122 response.status = '401 Unauthorized' 1123 response.Write('</head>\n<body><img src="/img/header_error.jpg"><h1>401 Unauthorized</h1>') 1124 response.Write('<br>%r<br>%r' % (error, request.path)) 1125 response.Write(' ' * 512) 1126 response.Write('%s</body></html>' % self._TrackPage('401 Unauthorized', req_session)) 1127 else: 1128 l = request.DumpRequestToList() 1129 username = 'N/A' 1130 if hasattr(sess, 'esps'): 1131 if 'username' in sess.esps.contents: 1132 username = sess.esps.contents['username'] 1133 info = 'Username: %s, address: %s' % (username, request.ep.address) 1134 response.Clear() 1135 response.Write('<html><head>') 1136 response.Write('\t<title>501 Internal Server Error</title>') 1137 response.Write('\t<link rel="stylesheet" href="/lib/error.css">') 1138 response.status = '501 Internal Server Error' 1139 response.Write('</head>\n<body><img src="/img/header_error.jpg"><h1>501 Internal Server Error</h1>') 1140 logWarning = 0 1141 sqlerror = isinstance(error, SQLError) 1142 if sqlerror: 1143 if error.errorRecords: 1144 for er in error.errorRecords: 1145 response.Write('<font color=red size=4>%s</font><br><br>' % htmlwriter.Swing(str(er[0]))) 1146 1147 if error.errorRecords and error.errorRecords[0][2] == 50000 and error.errorRecords[0][4] == 13: 1148 log.LogException(severity=log.LGWARN) 1149 logWarning = 1 1150 response.Write('<pre>') 1151 response.Write(htmlwriter.Swing(''.join(traceback.format_exc()))) 1152 response.Write('</pre><b>The request:</b><hr><pre>') 1153 for s in l: 1154 response.Write(htmlwriter.Swing(s)) 1155 1156 response.Write('</pre>%s</body></html>' % self._TrackPage('501 Internal Server Error', req_session)) 1157 if not logWarning: 1158 log.LogException(info + '\nRequest=\n' + '\n'.join(l)) 1159 1160 def CheckSessionTimeout(self, sess, oldRequestCount): 1161 blue.pyos.synchro.SleepWallclock(60000 * self.TimoutOutIntervalInMinutes) 1162 try: 1163 if sess.requestCount == oldRequestCount: 1164 svclog.debug("%s's http session timeout, removing session. sid=%s", sess.esps.contents.get('username', '?'), sess.sid) 1165 if sess.sid != 0: 1166 self.OnSessionEnd(sess.sid) 1167 finally: 1168 del sess.esps.contents['timeoutTimer'] 1169 1170 @Memoize(DB_SETTING_CACHE_TIMEOUT) 1171 def _GetDownloadableFileExtensions(self): 1172 if macho.mode == 'server': 1173 cacheSvc = self.session.ConnectToService('cache') 1174 else: 1175 cacheSvc = self.session.ConnectToAnyService('cache') 1176 rawString = cacheSvc.Setting('zsystem', 'SP-Allowed-Extensions') 1177 if not rawString: 1178 return ['py', 1179 'gif', 1180 'jpg', 1181 'htm', 1182 'htc', 1183 'html', 1184 'txt', 1185 'mp3', 1186 'js', 1187 'xsl', 1188 'xml', 1189 'xls', 1190 'css', 1191 'png', 1192 'ico', 1193 'pickle', 1194 'lbw', 1195 'cab', 1196 '7z', 1197 'uc', 1198 'swf', 1199 'wav', 1200 'mp3', 1201 'mid', 1202 'midi', 1203 'aif', 1204 'wma', 1205 'ogg', 1206 'woff', 1207 'ttf', 1208 'woff2', 1209 'fon', 1210 'otf', 1211 'eot', 1212 'svg'] 1213 else: 1214 return [ i.strip() for i in rawString.split(',') ]