ExceptionWrapperGPCS.py (9714B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\script\net\ExceptionWrapperGPCS.py 2 from __future__ import print_function 3 import sys 4 import types 5 import zlib 6 import log 7 import traceback2 8 import blue 9 from carbon.common.lib import const 10 from carbon.common.script.net import machobase, machoNet, objectCaching 11 from carbon.common.script.net.machoNetExceptions import UnMachoDestination, MachoException, MachoWrappedException 12 from carbon.common.script.net.moniker import UpdateMoniker 13 from carbon.common.script.sys.serviceConst import ROLE_PROGRAMMER, ROLE_QA 14 from cluster import MACHONETERR_WRAPPEDEXCEPTION 15 from eveexceptions import UserError 16 from inventorycommon import WrongInventoryLocation 17 from eveprefs import prefs 18 import monolithsentry 19 from stringutil import strx 20 from machonet_tracing import IngressTracer 21 22 class ExceptionWrapper: 23 __guid__ = 'gpcs.ExceptionWrapper' 24 25 def __init__(self): 26 self.errornumber = 1 27 28 def GetErrorID(self): 29 if machoNet.mode == 'client': 30 theID = '{%s #%d}' % (machoNet.mode, self.errornumber) 31 else: 32 theID = '{%s-%s-%d #%d}' % (machoNet.mode, 33 self.machoNet.GetLocalHostName(), 34 self.machoNet.GetNodeID(), 35 self.errornumber) 36 self.errornumber += 1 37 return theID 38 39 def CallUp(self, packet): 40 try: 41 try: 42 return self.ForwardCallUp(packet) 43 except (UnMachoDestination, 44 MachoException, 45 objectCaching.CacheOK, 46 UpdateMoniker, 47 UserError) as e: 48 if isinstance(e, objectCaching.CacheOK): 49 IngressTracer.set_object_cache_ok() 50 if isinstance(e, UnMachoDestination) and getattr(packet.destination, 'nodeID', None): 51 if packet.destination.nodeID < const.maxNodeID and machoNet.mode == 'proxy': 52 self.machoNet.HandleUnMachoDestination(packet.destination.nodeID, packet) 53 return packet.ErrorResponse(MACHONETERR_WRAPPEDEXCEPTION, (machobase.DumpsSanitized(e),)) 54 except Exception as e: 55 exc_info = sys.exc_info() 56 source = 'server' if hasattr(packet.source, 'nodeID') else 'client' 57 monolithsentry.capture_exception('Failed to service remote call', exc_info, new_tags={'destination_service': packet.destination.service, 58 'macho_source': source, 59 'destination_method': packet.payload[1]}) 60 exctype, exc = exc_info[:2] 61 stack, serverStackKey = log.GetStack(traceback2.extract_tb(sys.exc_info()[2], extract_locals=1), show_locals=1) 62 dasID = self.GetErrorID() 63 desc = ['%s caught an exception while handling a remote call. The caller should send a traceback with ID %s, which hopefully arrive at the originating node. Server info follows:' % (machoNet.mode, dasID)] 64 desc += stack 65 desc += traceback2.format_exception_only(exctype, exc) 66 try: 67 memory = blue.sysinfo.GetMemory() 68 ram = memory.pageFile / 1024 / 1024 69 cpuLoad = self.machoNet.GetCPULoad() 70 memLeft = memory.availablePhysical / 1024 / 1024 71 txt = 'System Information: ' 72 txt += 'Total CPU load: %s%%' % int(cpuLoad) 73 txt += ' | Process memory in use: %s MB' % ram 74 txt += ' | Physical memory left: %s MB\n' % memLeft 75 desc.append(txt) 76 sessionInfo = 'session was ' + str(session) 77 desc.append(sessionInfo) 78 except Exception as e: 79 sys.exc_clear() 80 81 self.machoNet.LogError('\n'.join(desc)) 82 session.LogSessionHistory('An exception occurred while handling a remote call. Traceback ID=%s' % dasID) 83 session.hasproblems = 1 84 sm.GetService('alert').SendStackTraceAlert(serverStackKey, ''.join(desc), 'Delivering Error To Remote Host') 85 variables = [] 86 if machoNet.mode == 'client': 87 variables += ['Context info logged on the client, session=%s\n' % (strx(session),)] 88 elif prefs.clusterMode in ('LOCAL', 'MASTER', 'TRANSLATION') or session and 0 != session.role & (ROLE_QA | ROLE_PROGRAMMER): 89 variables += ['Context info logged on %s node %d, host=%s\n' % (machoNet.mode, self.machoNet.GetNodeID(), self.machoNet.GetLocalHostName())] 90 if isinstance(e, WrongInventoryLocation): 91 e = RuntimeError(e.args) 92 else: 93 variables = [] 94 if isinstance(e, WrongInventoryLocation): 95 item = e.args[-1] 96 itemInfo = item[:2] + ['owner redacted', 'location redacted'] + item[-7:] 97 exceptionInfo = e.args[:-1] + tuple(itemInfo) 98 e = RuntimeError(*exceptionInfo) 99 stack = ['//%s/host=%s/nodeID=%d/errorHashKey=%d\n' % (machoNet.mode, 100 self.machoNet.GetLocalHostName(), 101 self.machoNet.GetNodeID(), 102 serverStackKey[0])] 103 return packet.ErrorResponse(MACHONETERR_WRAPPEDEXCEPTION, (machobase.DumpsSanitized(e), (1, 104 stack, 105 variables, 106 dasID, 107 serverStackKey, 108 self.machoNet.GetNodeID(), 109 machoNet.mode))) 110 111 except: 112 log.LogException('Could not create ErrorResponse, sending an empty one') 113 return packet.ErrorResponse(MACHONETERR_WRAPPEDEXCEPTION, None) 114 115 def CallDown(self, packet): 116 try: 117 return self.ForwardCallDown(packet) 118 except UnMachoDestination as e: 119 if getattr(packet.destination, 'nodeID', None): 120 if packet.destination.nodeID < const.maxNodeID and machoNet.mode == 'server': 121 self.machoNet.HandleUnMachoDestination(packet.destination.nodeID, packet) 122 raise 123 except MachoWrappedException as e: 124 serverStack = None 125 if e.payload and len(e.payload) > 1: 126 clientStack, clientStackKey = log.GetStack(traceback2.extract_stack()) 127 serverStack = e.payload[1][1] 128 serverStackVariables = e.payload[1][2] 129 fullReport = ['<-- top of local (%s) info -->\n' % machoNet.mode] 130 fullReport += clientStack 131 fullReport.append('<-- here we crossed the wire -->\n') 132 fullReport += serverStack 133 fullReport += serverStackVariables 134 fullReport.append('<-- bottom of remote (%s) info -->\n' % e.payload[1][6]) 135 if e.payload[1][0]: 136 logfunc = self.machoNet.LogError 137 m = 'Error' 138 else: 139 logfunc = self.machoNet.LogWarn 140 m = 'Warning' 141 fullReport = ''.join(fullReport) 142 logfunc('Exception traceback, origin %s:\n%s' % (e.payload[1][3], fullReport)) 143 serverStackKey = e.payload[1][4] 144 combinedStack = clientStackKey[1] + serverStackKey[1] 145 combinedStackKey = [zlib.adler32(combinedStack), combinedStack] 146 sm.GetService('alert').SendStackTraceAlert(combinedStackKey, fullReport, m, nextErrorKeyHash=serverStackKey[0], nodeID=e.payload[1][5]) 147 if machoNet.mode == 'client': 148 windowStream = log.MsgWindowStream() 149 if windowStream: 150 print('Exception Traceback-', file=windowStream) 151 print(fullReport, end=' ', file=windowStream) 152 windowStream.flush() 153 newException = machobase.LoadsSanitized(e.payload[0]) 154 if prefs.clusterMode in ('LOCAL', 'TEST', 'MASTER'): 155 if 'proxy' == machoNet.mode: 156 newException.serverStack = serverStack 157 raise newException 158 159 def NotifyUp(self, packet): 160 try: 161 self.ForwardNotifyUp(packet) 162 except Exception: 163 log.LogException('Exception during remote notification') 164 165 def FormatException(self, etype, value): 166 ret = [] 167 if type(etype) == types.ClassType: 168 stype = etype.__name__ 169 else: 170 stype = etype 171 ret.append('Exception:') 172 if value is None: 173 ret.append(strx(stype)) 174 else: 175 if etype is SyntaxError: 176 try: 177 msg, (filename, lineno, offset, line) = value 178 except Exception: 179 sys.exc_clear() 180 181 if line is None: 182 line = '<unknown>' 183 else: 184 line = line.strip() 185 item = '%s(%d) syntax error at position %s, of code:\t%s' % (filename, 186 lineno, 187 offset, 188 line) 189 ret.append(item) 190 pos = item.find(':') 191 item = ' ' * pos + ' ' + ' ' * offset + '^' 192 s = self._some_str(value) 193 if s: 194 ret.append('%s: %s' % (strx(stype), s)) 195 else: 196 ret.append(strx(stype)) 197 return ret 198 199 def _some_str(self, value): 200 try: 201 return strx(value) 202 except Exception: 203 return '<unprintable %s object>' % type(value).__name__