local_error_spike_report.py (1579B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\monolithsentry\client\local_error_spike_report.py 2 import logging, blue, monolithsentry 3 from carbon.common.lib.const import SEC 4 logger = logging.getLogger('monolithsentry.local_spike') 5 _error_record = {} 6 DEFAULT_ERROR_THRESHOLD = 5 7 DEFAULT_ERROR_TTL_SECONDS = 300 8 DEFAULT_SENTRY_COOLDOWN_SECONDS = 600 9 10 def record_handled_failure(error_key, message, threshold = DEFAULT_ERROR_THRESHOLD, error_ttl = DEFAULT_ERROR_TTL_SECONDS, sentry_cooldown = DEFAULT_SENTRY_COOLDOWN_SECONDS): 11 current_time = blue.os.GetWallclockTime() 12 last_error_time, count, last_sentry_report_time = _error_record.get(error_key, (current_time, 0, 0)) 13 elapsed_seconds = (current_time - last_error_time) / SEC 14 if elapsed_seconds > error_ttl: 15 count = 0 16 count += 1 17 last_error_time = current_time 18 _error_record[error_key] = (last_error_time, count, last_sentry_report_time) 19 if count <= threshold: 20 return 21 elapsed_sentry_seconds = (current_time - last_sentry_report_time) / SEC 22 if last_sentry_report_time and elapsed_sentry_seconds < sentry_cooldown: 23 return 24 sentry_message = '%s - local client error spike: key=%s count=%s within %ss (threshold=%s)' % (message, 25 error_key, 26 count, 27 error_ttl, 28 threshold) 29 monolithsentry.capture_error(sentry_message, extra={'error_key': error_key, 30 'error_count': count, 31 'error_ttl': error_ttl}, new_tags={'client_error_key': error_key}) 32 _error_record[error_key] = (current_time, 0, current_time)