alert.py (43336B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\script\sys\alert.py 2 from __future__ import print_function 3 import base64 4 import cStringIO 5 import copy 6 import gzip 7 import smtplib 8 import sys 9 import types 10 import zlib 11 import blue 12 import eve.common.lib.appConst as const 13 import log 14 import stackless 15 import uthread 16 import utillib 17 from carbon.common.script.net import machoNet 18 from carbon.common.script.sys.service import Service 19 from carbon.common.script.sys.serviceConst import ROLE_ANY, ROLE_PLAYER, ROLE_SERVICE, SERVICE_RUNNING, SERVICE_START_PENDING, SERVICE_STOP_PENDING, SERVICE_STOPPED 20 from carbon.common.script.util.format import FmtAmtEng, FmtDateEng, FmtTimeEng 21 from cluster import SERVICE_POLARIS 22 from eveexceptions import ExceptionEater, UnmarshalError 23 from eveprefs import prefs, boot 24 LOCATION_TQ = 1 25 LOCATION_BFC = 2 26 LOCATION_CCP = 3 27 LOCATION_OTHER = 4 28 ORIGIN_UNKNOWN = 0 29 ORIGIN_CLIENT = 1 30 ORIGIN_PROXY = 2 31 ORIGIN_SERVER = 3 32 LOGGING_BATCHED = 'B' 33 LOGGING_DETAILS = 'D' 34 MAX_DETAILED_USER_ERRORS_PER_HOUR = 50 35 originLabels = {ORIGIN_UNKNOWN: 'unknown', 36 ORIGIN_CLIENT: 'client', 37 ORIGIN_PROXY: 'proxy', 38 ORIGIN_SERVER: 'server'} 39 BEAN_DELIVERY_TIME = 15 40 41 class Alert(Service): 42 __guid__ = 'svc.alert' 43 __displayname__ = 'Alert Service' 44 __exportedcalls__ = {'Alert': [ROLE_SERVICE], 45 'SendSimpleEmailAlert': [ROLE_SERVICE], 46 'SendMail': [ROLE_SERVICE], 47 'SendProxyStackTraceAlert': [ROLE_SERVICE], 48 'SendClientStackTraceAlert': [ROLE_PLAYER], 49 'BeanCount': [ROLE_ANY], 50 'BeanDelivery': [ROLE_ANY], 51 'GroupBeanDelivery': [ROLE_ANY], 52 'GetCPULoad': [ROLE_SERVICE], 53 'LogModeChanged': [ROLE_SERVICE], 54 'GetLogModeForError': [ROLE_ANY]} 55 if boot.role in ('server',): 56 __startupdependencies__ = ['DB2', 'machoNet'] 57 elif boot.role in ('client', 'proxy'): 58 __startupdependencies__ = ['machoNet'] 59 else: 60 raise RuntimeError('Unkown boot role: ' + boot.role) 61 __dependencies__ = ['machoNet'] 62 __notifyevents__ = ['OnBeanPrime'] 63 64 def Run(self, memStream = None): 65 self.throttles = {} 66 self.stacktraceLogMode = {} 67 self.stacktracebeancounts = {} 68 self.stacktracebeangroupcounts = {} 69 self.stackTraceUserCount = {} 70 self.healthStats = utillib.KeyVal(nextBeanDelivery=0, totalBeanCount=0, totalErrorCount=0, lastBeanCount=0, lastErrorCount=0) 71 self.schema = None 72 self.computername = blue.pyos.GetEnv().get('COMPUTERNAME', '?') 73 self.mail_defaultrecpt = prefs.GetValue('alertEmailAddress', '').split(';') 74 domain = blue.pyos.GetEnv().get('USERDNSDOMAIN', '?').upper() 75 if domain == 'EVE.COM': 76 self.location = LOCATION_TQ 77 location = ' Tranquility' 78 self.mail_defaultrecpt = prefs.GetValue('alertEmailAddress', 'stacktraces@ccpgames.com').split(';') 79 elif domain == 'SISI.EVE.LOCAL': 80 self.location = LOCATION_BFC 81 location = ' Singularity' 82 elif domain == 'CCP.AD.LOCAL': 83 self.location = LOCATION_CCP 84 location = ' CCP %s' % domain 85 else: 86 self.location = LOCATION_OTHER 87 location = '' 88 self.mail_sender = 'Alert Service on%s %s <alerts@ccpgames.com>' % (location, self.computername) 89 self.mail_queue = uthread.Queue() 90 self.mail_server = None 91 self.tasklets = [] 92 self.bytesSent = 0 93 self.beanServerWrites = 0 94 self.groupBeanServerWrites = 0 95 self.isLiveCluster = None 96 self.errorTypes = {'Info': 'I', 97 'Warning': 'W', 98 'Error': 'E', 99 'Delivering Error To Remote Host': 'D'} 100 self.origins = {ORIGIN_SERVER: 'S', 101 ORIGIN_PROXY: 'P', 102 ORIGIN_CLIENT: 'C'} 103 self.tasklets.append(uthread.new(self.__mailqueue)) 104 self.tasklets.append(uthread.new(self.__BeanDeliveryBoy)) 105 self.LogInfo('Alert service will deliver errors up to the next tier every', prefs.GetValue('beanDeliveryTime', BEAN_DELIVERY_TIME), 'minutes (prefs.beanDeliveryTime)') 106 107 def Stop(self, stream): 108 r = Service.Stop(self, stream) 109 for t in self.tasklets: 110 t.kill() 111 112 return r 113 114 def IsLiveCluster(self): 115 if self.isLiveCluster is None and sm.IsServiceRunning('cache'): 116 cacheSvc = sm.GetService('cache') 117 if boot.role == 'server' and prefs.clusterMode == 'LIVE' and cacheSvc.Setting('zcluster', 'DB_NAME') == cacheSvc.Setting('zcluster', 'Database'): 118 self.isLiveCluster = True 119 else: 120 self.isLiveCluster = False 121 self.LogInfo('This is not a live cluster. Will use detailed logMode for errors.') 122 return self.isLiveCluster 123 124 def DbZCluster(self): 125 if self.schema is not None: 126 return self.schema 127 if machoNet.mode != 'client': 128 self.schema = self.DB2.GetSchema('zcluster') 129 return self.schema 130 self.LogError('No DB on Client! ') 131 132 def __GetMailServer(self): 133 if not self.mail_server: 134 try: 135 local_hostname = blue.pyos.GetEnv().get('COMPUTERNAME', '?') 136 self.mail_server = None 137 server = None 138 if hasattr(prefs, 'alertEmailServer'): 139 server = str(prefs.alertEmailServer) 140 self.LogInfo('Alert service using mail server from prefs.ini:', prefs.alertEmailServer) 141 elif self.location == LOCATION_TQ: 142 server = '10.210.10.251' 143 self.LogInfo('Alert service using default TQ mailserver', server) 144 elif self.location == LOCATION_BFC: 145 server = '10.210.10.251' 146 self.LogInfo('Alert service using default BFC mailserver', server) 147 elif self.location == LOCATION_CCP: 148 server = 'rkv-it-exch.ccp.ad.local' 149 self.LogInfo('Alert service using default CCP mailserver', server) 150 if server: 151 if sys.version_info[:3] == (2, 2, 1): 152 self.mail_server = smtplib.SMTP(server) 153 else: 154 self.mail_server = smtplib.SMTP(server, local_hostname=local_hostname) 155 except Exception: 156 log.LogTraceback('Mail server not working.', severity=log.LGWARN) 157 sys.exc_clear() 158 self.mail_server = None 159 160 return self.mail_server 161 162 def Alert(self, sender, subject, message, throttle = None, recipients = None, html = 0, sysinfo = 0, attachments = []): 163 if throttle: 164 if sender in self.throttles and self.throttles[sender] > blue.os.GetWallclockTime(): 165 return self.throttles[sender] 166 self.throttles[sender] = blue.os.GetWallclockTime() + throttle 167 return self.SendSimpleEmailAlert(message=message, subject=subject, sysinfo=sysinfo, html=html, recipients=recipients, attachments=attachments) 168 169 def GetSysInfo(self): 170 pyinfo = [('pythonver', sys.version)] 171 env = blue.pyos.GetEnv() 172 created = blue.sysinfo.processStartTime 173 kernel = blue.sysinfo.GetProcessTimes().systemTime 174 user = blue.sysinfo.GetProcessTimes().userTime 175 cputime = kernel + user 176 uptime = blue.os.GetWallclockTime() - created 177 memInfo = blue.sysinfo.GetMemory() 178 workingSetSize = memInfo.workingSet 179 pagefileUsage = memInfo.pageFile 180 ram = 1 + memInfo.totalPhysical / 1024 / 1024 181 hostinfo = [('hostname', env.get('COMPUTERNAME', '?')), 182 ('domain', env.get('USERDNSDOMAIN', '?')), 183 ('cpuid', env.get('PROCESSOR_IDENTIFIER', '?')), 184 ('cpucount', env.get('NUMBER_OF_PROCESSORS', '?')), 185 ('cpuspeed', '%s MHz' % blue.sysinfo.cpu.frequency), 186 ('starttime', FmtDateEng(created)), 187 ('uptime', FmtTimeEng(uptime)), 188 ('cputime', FmtTimeEng(cputime)), 189 ('kerneltime', FmtTimeEng(kernel)), 190 ('usertime', FmtTimeEng(user)), 191 ('currentcpu', '%.1f%%' % (self.GetCPULoad(35) * 100.0)), 192 ('ram', '%s MB' % ram), 193 ('memusage', '%s K' % FmtAmtEng(workingSetSize / 1024)), 194 ('vmsize', '%s K' % FmtAmtEng(pagefileUsage / 1024)), 195 ('nodeid', sm.services['machoNet'].nodeID)] 196 buildinfo = [('version', boot.version), 197 ('build', boot.build), 198 ('codename', str(boot.codename)), 199 ('region', str(boot.region)), 200 ('role', machoNet.mode)] 201 return pyinfo + hostinfo + buildinfo 202 203 def BeanCount(self, stackIDHash): 204 errorID, logMode = self.stacktraceLogMode.get(stackIDHash, (None, None)) 205 if machoNet.mode == 'proxy' or errorID is None: 206 return (errorID, logMode) 207 for i in range(30): 208 errorID, logMode = self.stacktraceLogMode[stackIDHash] 209 if errorID is None: 210 blue.pyos.synchro.SleepWallclock(5000) 211 continue 212 break 213 214 if errorID is not None: 215 if logMode == LOGGING_BATCHED: 216 self.stacktracebeancounts[errorID] = (self.stacktracebeancounts.get(errorID, [0])[0] + 1, blue.os.GetWallclockTime()) 217 else: 218 self.LogError('Invalid Bean Counting mode: ', logMode) 219 return (errorID, logMode) 220 221 def OnClusterStartup(self): 222 if machoNet.mode != 'server': 223 return 224 beans = self.DbZCluster().Errors_Prime() 225 self.OnBeanPrime(beans) 226 if all([not self.machoNet.IsResurrectedNode(), self.machoNet.GetNodeID() == self.machoNet.GetNodeFromAddress(SERVICE_POLARIS, 0)]): 227 self.machoNet.ProxyBroadcast('OnBeanPrime', beans) 228 229 def OnBeanPrime(self, beans): 230 if type(beans) == types.DictType: 231 self.stacktraceLogMode.update(beans) 232 else: 233 for bean in beans: 234 self.stacktraceLogMode[bean.errorKeyHash] = (bean.errorID, bean.logMode) 235 236 def ConvertFloodingGroupBeansToBeans(self, floodingErrors): 237 if len(floodingErrors) == 0: 238 return 239 groupData = self.stacktracebeangroupcounts 240 toDelete = [] 241 for errorID, usersData in groupData.iteritems(): 242 if errorID not in floodingErrors: 243 continue 244 for userID, errorData in usersData.iteritems(): 245 if errorID in self.stacktracebeancounts: 246 self.stacktracebeancounts[errorID] = (self.stacktracebeancounts.get(errorID, [0])[0] + errorData[0], min(blue.os.GetWallclockTime(), max(self.stacktracebeancounts.get(errorID, [0])[1], errorData[1]))) 247 else: 248 self.stacktracebeancounts[errorID] = (errorData[0], errorData[1]) 249 250 toDelete.append(errorID) 251 252 for errorID in toDelete: 253 del self.stacktracebeangroupcounts[errorID] 254 255 def CompressGroupBeans(self, groupBeans): 256 res = zlib.compress(blue.marshal.Save(groupBeans)) 257 return res 258 259 def ExpandGroupBeans(self, compressedGroupBeans): 260 if isinstance(compressedGroupBeans, dict): 261 return compressedGroupBeans 262 res = None 263 try: 264 res = blue.marshal.Load(zlib.decompress(compressedGroupBeans)) 265 except (UnmarshalError, zlib.error): 266 firstBit = '<error encoding>' 267 with ExceptionEater('Encoding some badness in the beancounter'): 268 firstBit = base64.b64encode(compressedGroupBeans[:1000]) 269 self.LogError('BeanDelivery recieved a non-standard bean payload, likely not exefile in origin. Session:', repr(session), 'First 1000char of compressed payload:', firstBit) 270 log.LogException('BeanDelivery recieved a non-standard bean payload') 271 with ExceptionEater('Reporting BeanDelivery error to Alert'): 272 message = 'The following session sent us a set of Beans that are probably all attacky:\n%s\n\n' % (repr(session),) 273 message += 'The compressed payload follows (base64 encoded):\n%s' % (base64.b64encode(compressedGroupBeans),) 274 self.Alert('BeanDelivery', 'Non-standard BeanCounting payload', message) 275 276 return res 277 278 def __BeanDeliveryBoy(self): 279 while self.state == SERVICE_RUNNING: 280 beanDeliveryTime = prefs.GetValue('beanDeliveryTime', BEAN_DELIVERY_TIME) 281 self.healthStats.nextBeanDelivery = blue.os.GetWallclockTime() + beanDeliveryTime * const.MIN 282 blue.pyos.synchro.SleepWallclock(beanDeliveryTime * 60000) 283 try: 284 self.CheckAndExpireUserCounts() 285 startTime = blue.os.GetWallclockTime() 286 try: 287 self.healthStats.lastBeanCount = 0 288 self.healthStats.lastErrorCount = 0 289 for errorID in self.stacktracebeancounts.keys(): 290 errorCount, lastDate = self.stacktracebeancounts[errorID] 291 self.healthStats.lastBeanCount += 1 292 self.healthStats.lastErrorCount += errorCount 293 self.healthStats.totalBeanCount += 1 294 self.healthStats.totalErrorCount += errorCount 295 296 except: 297 raise 298 299 if self.stacktracebeancounts or self.stacktracebeangroupcounts: 300 self.LogInfo(machoNet.mode, 'Delivering beans. been count', len(self.stacktracebeancounts), ' group bean count', len(self.stacktracebeangroupcounts)) 301 if machoNet.mode == 'client': 302 floodingErrors = self.UpdateLogModes() 303 self.ConvertFloodingGroupBeansToBeans(floodingErrors) 304 beans = self.stacktracebeancounts 305 self.stacktracebeancounts = {} 306 self.bytesSent += len(str(beans)) 307 sm.ProxySvc('alert').BeanDelivery(beans) 308 groupBeans = self.stacktracebeangroupcounts 309 self.stacktracebeangroupcounts = {} 310 compressedBeans = self.CompressGroupBeans(groupBeans) 311 self.bytesSent += len(str(compressedBeans)) 312 sm.ProxySvc('alert').GroupBeanDelivery(compressedBeans) 313 elif machoNet.mode == 'proxy': 314 beans = self.stacktracebeancounts 315 self.stacktracebeancounts = {} 316 if getattr(self, 'solalert', None) is None: 317 self.solalert = self.session.ConnectToSolServerService('alert') 318 self.bytesSent += len(str(beans)) 319 self.solalert.BeanDelivery(beans) 320 groupBeans = self.stacktracebeangroupcounts 321 self.stacktracebeangroupcounts = {} 322 compressedBeans = self.CompressGroupBeans(groupBeans) 323 self.bytesSent += len(str(compressedBeans)) 324 self.solalert.GroupBeanDelivery(compressedBeans) 325 elif machoNet.mode == 'server': 326 polarisNodeID = self.machoNet.GetNodeFromAddress(SERVICE_POLARIS, 0) 327 if self.machoNet.GetNodeID() != polarisNodeID: 328 beans = self.stacktracebeancounts 329 self.stacktracebeancounts = {} 330 if getattr(self, 'polarisalert', None) is None: 331 self.polarisalert = self.session.ConnectToSolServerService('alert', polarisNodeID) 332 self.bytesSent += len(str(beans)) 333 self.polarisalert.BeanDelivery(beans) 334 groupBeans = self.stacktracebeangroupcounts 335 self.stacktracebeangroupcounts = {} 336 compressedBeans = self.CompressGroupBeans(groupBeans) 337 self.bytesSent += len(str(compressedBeans)) 338 self.polarisalert.GroupBeanDelivery(compressedBeans) 339 else: 340 for errorID in self.stacktracebeancounts.keys(): 341 errorCount, lastDate = self.stacktracebeancounts[errorID] 342 if lastDate < startTime - const.DAY or lastDate > startTime: 343 lastDate = startTime 344 del self.stacktracebeancounts[errorID] 345 self.DbZCluster().Errors_Update(errorID, errorCount, lastDate) 346 self.beanServerWrites += 1 347 348 nodeID = self.machoNet.GetNodeID() 349 beanGroups = self.stacktracebeangroupcounts 350 self.stacktracebeangroupcounts = {} 351 for errorID, userReports in beanGroups.iteritems(): 352 for userID, data in userReports.iteritems(): 353 cnt, tm, charID, locationID1, locationID2, argsStr, errNodeID = data 354 if type(argsStr) == types.UnicodeType: 355 argsStr = argsStr.encode('ascii', 'replace') 356 if userID > 0: 357 self.DbZCluster().ErrorUserDates_Update(errorID, userID, charID, cnt, argsStr, errNodeID, locationID1, locationID2) 358 self.groupBeanServerWrites += 1 359 blue.pyos.BeNice() 360 361 else: 362 self.LogError('WTF??? ', machoNet.mode, '????') 363 self.LogInfo('|', machoNet.mode, '| Delivery of beans #Bytes:', self.bytesSent, '# DB bean writes', self.beanServerWrites, '# DB group writes', self.groupBeanServerWrites, ' time (ms):', blue.os.TimeDiffInMs(startTime, blue.os.GetWallclockTime())) 364 self.bytesSent = 0 365 self.bytesSent = 0 366 self.groupBeanServerWrites = 0 367 except Exception as e: 368 self.LogError('An exception has occurred: ', e) 369 raise 370 sys.exc_clear() 371 372 def BeanDelivery(self, beans): 373 for errorID, v in beans.iteritems(): 374 c, l = v 375 if errorID in self.stacktracebeancounts: 376 self.stacktracebeancounts[errorID] = (self.stacktracebeancounts.get(errorID, [0])[0] + c, min(blue.os.GetWallclockTime(), max(self.stacktracebeancounts.get(errorID, [0])[1], l))) 377 else: 378 self.stacktracebeancounts[errorID] = v 379 380 def GroupBeanDelivery(self, compressedGroupBeans, nodeID = None): 381 groupBeans = self.ExpandGroupBeans(compressedGroupBeans) 382 for errorID, userReports in groupBeans.iteritems(): 383 for userID, data in userReports.iteritems(): 384 if errorID not in self.stacktracebeangroupcounts: 385 self.stacktracebeangroupcounts[errorID] = {} 386 cnt, tm, charID, locationID1, locationID2, argsStr, errNodeID = data 387 if userID in self.stacktracebeangroupcounts[errorID]: 388 currentCount = self.stacktracebeangroupcounts[errorID].get(userID, [0])[0] 389 currentTime = min(blue.os.GetWallclockTime(), max(self.stacktracebeangroupcounts[errorID].get(userID, [0])[1], tm)) 390 self.stacktracebeangroupcounts[errorID][userID] = (currentCount + cnt, 391 currentTime, 392 charID, 393 locationID1, 394 locationID2, 395 argsStr, 396 errNodeID or self.machoNet.GetNodeID()) 397 else: 398 self.stacktracebeangroupcounts[errorID][userID] = (cnt, 399 tm, 400 charID, 401 locationID1, 402 locationID2, 403 argsStr, 404 errNodeID or self.machoNet.GetNodeID()) 405 if machoNet.mode == 'proxy': 406 if errorID in self.stackTraceUserCount: 407 self.stackTraceUserCount[errorID].users.add(userID) 408 else: 409 self.stackTraceUserCount[errorID] = utillib.KeyVal(timestamp=blue.os.GetWallclockTime(), users=set([userID])) 410 blue.pyos.BeNice() 411 412 def SendStackTraceAlert(self, stackID, stackTrace, mode, **kw): 413 if getattr(self, 'state', SERVICE_START_PENDING) != SERVICE_RUNNING: 414 return 415 if 'machoNet' not in sm.services: 416 return 417 if machoNet.mode == 'client': 418 if not sm.services['machoNet'].IsConnected(): 419 return 420 elif sm.GetService('machoNet').IsClusterShuttingDown(): 421 return 422 f = stackless.getcurrent().frame 423 try: 424 while f: 425 f = f.f_back 426 if f and f.f_code.co_filename.endswith('alert.py') and f.f_code.co_name == '__SendStackTraceAlert': 427 return 428 429 finally: 430 f = None 431 432 uthread.pool('AlertSvc::__SendStackTraceAlert', self.SendStackTraceAlert_thread, stackID, stackTrace, mode, kw) 433 434 def SendStackTraceAlert_thread(self, stackID, stackTrace, mode, kw): 435 if getattr(self, 'state', SERVICE_START_PENDING) != SERVICE_RUNNING: 436 return 437 try: 438 if machoNet.mode != 'client' and 'nodeID' not in kw: 439 kw = copy.copy(kw) 440 kw['nodeID'] = self.machoNet.GetNodeID() 441 self.__SendStackTraceAlert(stackID, stackTrace, mode, **kw) 442 except Exception: 443 log.LogException('Exception in Error Logging, preventing infinite recursion', toAlertSvc=0) 444 sys.exc_clear() 445 446 def SendProxyStackTraceAlert(self, stackID, stackTrace, mode, **kw): 447 if 'origin' not in kw: 448 kw['origin'] = ORIGIN_PROXY 449 return self.__SendStackTraceAlert(stackID, stackTrace, mode, **kw) 450 451 def SendClientStackTraceAlert(self, stackID, stackTrace, mode, nextErrorKeyHash = None): 452 userID, charID, locationID1, locationID2 = self._GetSessionInfo() 453 return self.__SendStackTraceAlert(stackID, stackTrace, mode, userID=userID, nextErrorKeyHash=nextErrorKeyHash, origin=ORIGIN_CLIENT) 454 455 def __SendStackTraceAlert(self, stackID, stackTrace, mode, nodeID = None, userID = None, nextErrorKeyHash = None, origin = ORIGIN_SERVER): 456 457 def GetArgumentsFromStackTrace(stackTrace): 458 argPos = stackTrace.find('Arguments:') 459 if argPos > 0: 460 return stackTrace[argPos:] 461 return '' 462 463 uthread.Lock(self, 'SendStackTraceAlert', stackID[0]) 464 try: 465 if machoNet.mode == 'client': 466 if stackID[0] not in self.stacktraceLogMode: 467 self.LogInfo(machoNet.mode, "Don't know what beancounting method to use, crossing the wire.") 468 errorID, logMode = sm.ProxySvc('alert').BeanCount(stackID[0]) 469 if errorID is None: 470 self.LogInfo(machoNet.mode, 'The proxy wants a trace.') 471 self.bytesSent += len(str(stackID)) + len(str(stackTrace)) 472 self.stacktraceLogMode[stackID[0]] = sm.ProxySvc('alert').SendClientStackTraceAlert(stackID, stackTrace, mode, nextErrorKeyHash) 473 return 474 self.stacktraceLogMode[stackID[0]] = (errorID, logMode) 475 for i in range(30): 476 errorID, logMode = self.stacktraceLogMode[stackID[0]] 477 if errorID is None: 478 self.LogInfo(machoNet.mode, "we're already crossing the wire to acquire this dude's beancounting properties") 479 blue.pyos.synchro.SleepWallclock(5000) 480 continue 481 break 482 483 if errorID is None: 484 self.LogInfo(machoNet.mode, "Couldn't beancount it, so just bugger it.") 485 elif logMode == LOGGING_BATCHED: 486 self.stacktracebeancounts[errorID] = (self.stacktracebeancounts.get(errorID, [0])[0] + 1, blue.os.GetWallclockTime()) 487 elif logMode == LOGGING_DETAILS: 488 self.stacktracebeancounts[errorID] = (self.stacktracebeancounts.get(errorID, [0])[0] + 1, blue.os.GetWallclockTime()) 489 if errorID not in self.stacktracebeangroupcounts: 490 self.stacktracebeangroupcounts[errorID] = {} 491 sessionUserID, charID, locationID1, locationID2 = self._GetSessionInfo() 492 if userID is None: 493 userID = sessionUserID 494 argStr = GetArgumentsFromStackTrace(stackTrace) 495 if userID in self.stacktracebeangroupcounts[errorID]: 496 self.stacktracebeangroupcounts[errorID][userID] = (self.stacktracebeangroupcounts[errorID].get(userID, [0])[0] + 1, 497 blue.os.GetWallclockTime(), 498 charID, 499 locationID1, 500 locationID2, 501 argStr, 502 None) 503 else: 504 self.stacktracebeangroupcounts[errorID][userID] = (1, 505 blue.os.GetWallclockTime(), 506 charID, 507 locationID1, 508 locationID2, 509 argStr, 510 None) 511 else: 512 self.LogError('Strange... unknown bean counting method specified ', logMode, errorID) 513 elif machoNet.mode == 'proxy': 514 if getattr(self, 'solalert', None) is None: 515 self.solalert = self.session.ConnectToSolServerService('alert') 516 if nodeID and nodeID in self.machoNet.GetConnectedSolNodes(): 517 solalert = self.session.ConnectToSolServerService('alert', nodeID) 518 else: 519 solalert = self.solalert 520 try: 521 userID, charID, locationID1, locationID2 = self._GetSessionInfo() 522 if stackID[0] not in self.stacktraceLogMode: 523 self.LogInfo(machoNet.mode, "Don't know what beancounting method to use, crossing the wire.") 524 errorID, logMode = solalert.BeanCount(stackID[0]) 525 if errorID is None: 526 self.LogInfo(machoNet.mode, 'The sol node wants a trace.') 527 self.bytesSent += len(str(stackID)) + len(str(stackTrace)) 528 self.stacktraceLogMode[stackID[0]] = solalert.SendProxyStackTraceAlert(stackID, stackTrace, mode, nodeID=self.machoNet.GetNodeID(), userID=userID, nextErrorKeyHash=nextErrorKeyHash, origin=origin) 529 else: 530 self.stacktraceLogMode[stackID[0]] = (errorID, logMode) 531 else: 532 for i in range(30): 533 errorID, logMode = self.stacktraceLogMode[stackID[0]] 534 if errorID is None: 535 self.LogInfo(machoNet.mode, "we're already crossing the wire to acquire this dude's beancounting properties") 536 blue.pyos.synchro.SleepWallclock(5000) 537 continue 538 break 539 540 if errorID is None: 541 self.LogInfo(machoNet.mode, "Couldn't beancount it, so just bugger it.") 542 elif logMode == LOGGING_BATCHED: 543 self.LogInfo(machoNet.mode, 'Batched beancounting selected') 544 self.stacktracebeancounts[errorID] = (self.stacktracebeancounts.get(errorID, [0])[0] + 1, blue.os.GetWallclockTime()) 545 elif logMode == LOGGING_DETAILS: 546 self.LogInfo(machoNet.mode, 'Grouped beancounting selected') 547 self.stacktracebeancounts[errorID] = (self.stacktracebeancounts.get(errorID, [0])[0] + 1, blue.os.GetWallclockTime()) 548 argStr = GetArgumentsFromStackTrace(stackTrace) 549 ID = userID or 0 550 if errorID not in self.stacktracebeangroupcounts: 551 self.stacktracebeangroupcounts[errorID] = {} 552 if ID in self.stacktracebeangroupcounts[errorID]: 553 self.stacktracebeangroupcounts[errorID][ID] = (self.stacktracebeangroupcounts[errorID].get(ID, [0])[0] + 1, 554 blue.os.GetWallclockTime(), 555 charID, 556 locationID1, 557 locationID2, 558 argStr, 559 nodeID or self.machoNet.GetNodeID()) 560 else: 561 self.stacktracebeangroupcounts[errorID][ID] = (1, 562 blue.os.GetWallclockTime(), 563 charID, 564 locationID1, 565 locationID2, 566 argStr, 567 nodeID or self.machoNet.GetNodeID()) 568 else: 569 self.LogError('Strange... unknown bean counting method specified ', logMode) 570 except Exception as e: 571 self.LogError('Error during error handling:', e) 572 self.solalert = None 573 sys.exc_clear() 574 575 elif stackID[0] not in self.stacktraceLogMode: 576 k = stackID[1].encode('ascii', 'replace') 577 if type(stackTrace) == types.UnicodeType: 578 stackTrace = unicode(stackTrace).encode('ascii', 'replace') 579 self.beanServerWrites += 1 580 errorTypeChar = self.errorTypes.get(mode, 'U') 581 originChar = self.origins.get(origin, 'U') 582 if self.IsLiveCluster(): 583 logMode = 'B' 584 else: 585 logMode = 'D' 586 r = self.DbZCluster().Errors_Insert(k[:4000], stackTrace, stackID[0], nextErrorKeyHash, errorTypeChar, originChar, logMode)[0] 587 self.stacktraceLogMode[stackID[0]] = (r.errorID, r.logMode) 588 self.machoNet.ClusterBroadcast('OnBeanPrime', {stackID[0]: (r.errorID, r.logMode)}) 589 else: 590 errorID, logMode = self.stacktraceLogMode[stackID[0]] 591 if logMode == LOGGING_BATCHED: 592 self.BeanDelivery({errorID: (1, blue.os.GetWallclockTime())}) 593 elif logMode == LOGGING_DETAILS: 594 self.BeanDelivery({errorID: (1, blue.os.GetWallclockTime())}) 595 argStr = GetArgumentsFromStackTrace(stackTrace) 596 ID = userID or 0 597 self.GroupBeanDelivery({errorID: {ID: (1, 598 blue.os.GetWallclockTime(), 599 0, 600 0, 601 0, 602 argStr, 603 nodeID or self.machoNet.GetNodeID())}}) 604 else: 605 self.LogError('Strange... unknown bean counting method specified ', logMode) 606 if not sm.IsServiceRunning('machoNet'): 607 return 608 return self.stacktraceLogMode.get(stackID[0], (None, None)) 609 finally: 610 uthread.UnLock(self, 'SendStackTraceAlert', stackID[0]) 611 612 def SendSimpleEmailAlert(self, message, recipients = None, subject = None, sysinfo = 1, html = 0, attachments = [], subjectonly = 0): 613 self.LogInfo('SendSimpleEmailAlert ', (len(message), 614 recipients, 615 subject, 616 sysinfo, 617 html)) 618 if recipients is None: 619 recipients = self.mail_defaultrecpt 620 elif type(recipients) in types.StringTypes: 621 recipients = [recipients] 622 elif type(recipients) != types.ListType: 623 raise RuntimeError("SendSimpleEmailAlert: 'recipients' must be a list of strings") 624 if not recipients: 625 self.LogInfo('Not sending alert, no recipient specified', subject) 626 return 627 timestamp = FmtDateEng(blue.os.GetWallclockTime(), 'ns') 628 if subject is None: 629 subject = 'Server Alert: Alert from node %s on %s at %s' % (sm.services['machoNet'].nodeID, self.computername, timestamp) 630 elif subjectonly == 0: 631 subject = 'Server Alert: %s at %s' % (subject, timestamp) 632 if sysinfo: 633 if html: 634 message += '<br>\n<br>\nSysinfo:<br><table border=1>\n' 635 for k, v in self.GetSysInfo(): 636 message += '<tr><td>%s</td><td>%s</td></tr>\n' % (k, v) 637 638 else: 639 message += '\n\n' + '-' * 78 + '\n' 640 for k, v in self.GetSysInfo(): 641 message += '%-15s%s\n' % (k + ':', v) 642 643 from email.MIMEText import MIMEText 644 from email.MIMEMultipart import MIMEMultipart 645 from email.MIMEBase import MIMEBase 646 from email import encoders 647 msg = MIMEMultipart() 648 if html: 649 subtype = 'html' 650 charset = 'iso8859' 651 else: 652 subtype = 'plain' 653 charset = 'us-ascii' 654 if type(message) is unicode: 655 message = message.encode('utf-8') 656 charset = 'utf-8' 657 att = MIMEText(message, subtype, charset) 658 msg.attach(att) 659 subject = subject.replace('\n', '').replace('\r', '') 660 msg['Subject'] = subject 661 msg['From'] = self.mail_sender 662 msg['To'] = ', '.join(recipients) 663 for attName, attData in attachments: 664 io = cStringIO.StringIO() 665 zipfile = gzip.GzipFile(attName, 'w', 9, io) 666 zipfile.write(attData) 667 zipfile.close() 668 att = MIMEBase('application', 'gz') 669 att.set_payload(io.getvalue()) 670 encoders.encode_base64(att) 671 att.add_header('Content-Disposition', 'attachment', filename=attName + '.gz') 672 msg.attach(att) 673 674 self.SendMail(self.mail_sender, recipients, msg.as_string(0)) 675 676 def SendMail(self, *args, **kw): 677 channel = None 678 if kw.get('wait', False): 679 del kw['wait'] 680 channel = stackless.channel() 681 self.mail_queue.put((channel, args, kw)) 682 if channel: 683 return channel.receive() 684 685 def __mailqueue(self): 686 self.LogInfo('Alert: Starting Mail Queue thread') 687 try: 688 while self.state != SERVICE_STOPPED: 689 try: 690 channel, args, kw = self.mail_queue.get() 691 self.LogInfo('Alert: Got mail from Mail Queue') 692 srv = self.__GetMailServer() 693 if srv: 694 self.LogInfo('Alert: Sending mail') 695 srv.sendmail(*args, **kw) 696 self.LogInfo('Alert: Mail sent') 697 if channel: 698 channel.send(True) 699 else: 700 self.LogWarn('Alert: No mail server available, mail not sent. sendmail args=', args, 'kw=', kw) 701 if channel: 702 channel.send(False) 703 except smtplib.SMTPServerDisconnected as e: 704 self.mail_server = None 705 self.mail_queue.put((channel, args, kw)) 706 self.LogWarn('Failed to send alert, SMTP server disconnected. Will try again in 1 minute. SMTP Error: %s' % str(e)) 707 sys.exc_clear() 708 blue.pyos.synchro.SleepWallclock(60000) 709 except TaskletExit: 710 if 'channel' in locals() and channel is not None: 711 channel.send(False) 712 raise 713 except: 714 if 'channel' in locals() and channel is not None: 715 channel.send(False) 716 self.LogError('Alert: Unexpected error during mailqueue processing') 717 718 except TaskletExit: 719 if hasattr(self, 'state') and self.state not in (SERVICE_STOPPED, SERVICE_STOP_PENDING): 720 raise 721 finally: 722 self.LogInfo('Alert: Stopping Mail Queue thread') 723 724 def GetCPULoad(self, seconds = 300): 725 now = blue.os.GetWallclockTime() 726 then = now - seconds * const.SEC 727 total = 0 728 lastTime = now 729 i = len(blue.pyos.cpuUsage) - 1 730 while i > 0 and blue.pyos.cpuUsage[i].timestamp >= then: 731 total += blue.pyos.cpuUsage[i].userProcessCpuUsage 732 lastTime = blue.pyos.cpuUsage[i].timestamp 733 i -= 1 734 735 if lastTime == now: 736 return 0.0 737 return min(float(total) / (float(now) - float(lastTime)), 1.0) 738 739 def _GetSessionInfo(self): 740 if session: 741 return (session.userid, 742 session.charid, 743 None, 744 None) 745 return (None, None, None, None) 746 747 def NotifyAllSolAndProxyOfLogModeChange(self, errorID, logMode): 748 res = self.session.ConnectToAllServices('alert').LogModeChanged(errorID, logMode) 749 750 def LogModeChanged(self, errorID, logMode): 751 self.LogInfo(machoNet.mode, ' LogModeChanged ', errorID, ' to mode ', logMode) 752 for k, v in self.stacktraceLogMode.iteritems(): 753 if v[0] == errorID: 754 self.stacktraceLogMode[k] = (v[0], logMode) 755 756 def UpdateLogModes(self): 757 self.LogInfo(machoNet.mode, 'UpdateLogModes') 758 changes, flooded = sm.ProxySvc('alert').GetLogModeForError(self.stacktraceLogMode) 759 self.LogInfo(machoNet.mode, 'UpdateLogModes got changes =', changes) 760 self.LogInfo(machoNet.mode, 'UpdateLogModes got These Flooded =', flooded) 761 for errorHash, newLogMode in changes.iteritems(): 762 errorID = self.stacktraceLogMode[errorHash][0] 763 self.stacktraceLogMode[errorHash] = (errorID, newLogMode) 764 if newLogMode != LOGGING_DETAILS and errorID in self.stacktracebeangroupcounts: 765 del self.stacktracebeangroupcounts[errorID] 766 767 return flooded 768 769 def GetLogModeForError(self, errorIDs): 770 self.LogInfo(machoNet.mode, 'GetLogModeForError', self.stacktraceLogMode) 771 result = {} 772 flooded = [] 773 for errorID, logMode in errorIDs.iteritems(): 774 if errorID in self.stacktraceLogMode: 775 if logMode[1] != self.stacktraceLogMode[errorID][1]: 776 result[errorID] = self.stacktraceLogMode[errorID][1] 777 if logMode[1] == LOGGING_DETAILS: 778 if logMode[0] in self.stackTraceUserCount and len(self.stackTraceUserCount[logMode[0]].users) > MAX_DETAILED_USER_ERRORS_PER_HOUR: 779 flooded.append(logMode[0]) 780 781 return (result, flooded) 782 783 def CheckAndExpireUserCounts(self): 784 for errorID, userData in self.stackTraceUserCount.iteritems(): 785 if blue.os.TimeDiffInMs(userData.timestamp, blue.os.GetWallclockTime()) > const.HOUR / const.MSEC: 786 userData.timestamp = blue.os.GetWallclockTime() 787 userData.users = set() 788 789 def DoNastyLoggingTest(self, randomUser = False, maxLoops = 500000, showProgress = False): 790 if prefs.clusterMode not in ('LOCAL', 'TEST'): 791 print('BAD BAD BOY!, you are not to call this on a live server!!!') 792 return 793 self.tasklets.append(uthread.new(self.__DoNastyErrorLogSimulation, randomUser, maxLoops, showProgress)) 794 795 def __DoNastyErrorLogSimulation(self, randomUser = False, maxLoops = 50000, showProgress = False): 796 import random 797 if prefs.clusterMode not in ('LOCAL', 'TEST'): 798 print('BAD BAD BOY!, you are not to call this on a live server!!!') 799 return 800 if randomUser: 801 print('DoNastyErrorLogSimulation is generating many errors with !!RANDOM!! User IDs') 802 else: 803 print('DoNastyErrorLogSimulation is generatijng many errors with session User ID') 804 userIDStart = 10000 805 userIDRange = 25 806 cnt = 0 807 while self.state == SERVICE_RUNNING and cnt < maxLoops: 808 blue.pyos.BeNice() 809 errorKey = '============ TEST EXCEPTION %d ============' 810 errorStack = '\n TEST EXCEPTION %d logged at MM/DD/YYYY HH:MM:SS Unhandled exception in <TaskletExt object at 226a9588, abps=1001, ctxt=\'Tick:.SleepWallclock\'> \n Caught at: /common/lib/Fakebluepy.py(37) in CallWrapper \n Thrown at: /common/lib/Fakebluepy.py(24) in CallWrapper \n /common/lib/Fakenasty.py(1344) in OnFileModified_ \n /common/lib/Fakenasty.py(1595) in Bootstrap \n /common/lib/Fakenasty.py(2295) in ImportFromFile \n /common/lib/Fakenasty.py(468) in GetCode \n /common/lib/Fakenasty.py(517) in Compile \n /common/lib/Fakenasty.py(556) in Compile_int \n Exception: \n File "D:/depot/games/EVE-DEV/eve/server/script/../../../carbon/common/script/sys/alert.py", line 329 \n\n\n def SendStackTraceAlert(self, stackID, stackTrace, mode, **kw): \n ^ IndentationError: expected an indented block \n Arguments: \n self : <nasty.Compilor object at 0x037D5F88> \n pathname : \'script:/../../../carbon/common/script/sys/alert.py\' \n Locals: Lots! #%d\n Thread Locals: session was None System Information: Node ID: 1733 \n | Node Name: RobertPC | Total CPU load: 0%% | Process memory in use: 420 MB | Physical memory left: 213 MB \n ' 811 testUserID = None 812 cnt += 1 813 if randomUser: 814 testUserID = random.randint(userIDStart, userIDStart + userIDRange) 815 if cnt % 100000 == 0: 816 userIDStart += userIDRange 817 for i in range(10, 15): 818 self.SendStackTraceAlert((2000 + i, errorKey % i), errorStack % (i, cnt), 'error', userID=testUserID) 819 820 if showProgress: 821 print('.', end=' ') 822 if cnt % 100 == 0: 823 print('') 824 print('cnt = ', cnt) 825 826 print('============================== all done')