lbw.py (25053B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\script\util\lbw.py 2 from __future__ import print_function 3 import sys 4 import struct 5 import time 6 import re 7 import ctypes 8 import datetime 9 _c_counter_frequency = ctypes.c_int64() 10 ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(_c_counter_frequency)) 11 COUNTER_FREQUENCY = _c_counter_frequency.value 12 DELPHI_DATE_TIME_EPOCH = datetime.datetime(1899, 12, 30, 0, 0, 0) 13 LGINFO = 1 14 LGNOTICE = 32 15 LGWARN = 2 16 LGERR = 4 17 severity_name = {LGINFO: 'info', 18 LGNOTICE: 'notice', 19 LGWARN: 'warn', 20 LGERR: 'error'} 21 epoch_diff = 11644473600000L / 1000.0 22 P_BYTE = 2 23 P_DWORD = 3 24 P_INT = 4 25 P_LONGLONG = 8 26 P_SHORT_STRING = 6 27 P_LONG_STRING = 12 28 P_REAL = 17 29 P_SINGLE = None 30 31 def Read_BYTE(f): 32 return ord(f.read(1)) 33 34 35 def Read_SBYTE(f): 36 return struct.unpack('b', f.read(1))[0] 37 38 39 def Read_DWORD(f): 40 return struct.unpack('h', f.read(2))[0] 41 42 43 def Read_INT(f): 44 return struct.unpack('i', f.read(4))[0] 45 46 47 def Read_SHORT_STRING(f): 48 return f.read(Read_BYTE(f)) 49 50 51 def Read_LONG_STRING(f): 52 return f.read(Read_INT(f)) 53 54 55 def Read_REAL(f): 56 return struct.unpack('d', f.read(8))[0] 57 58 59 def Read_SINGLE(f): 60 return struct.unpack('f', f.read(4))[0] 61 62 63 def Read_LONGLONG(f): 64 return struct.unpack('q', f.read(8))[0] 65 66 67 def Write_BYTE(f, b): 68 f.write(chr(b)) 69 70 71 def Write_SBYTE(f, b): 72 f.write(struct.pack('b', b)) 73 74 75 def Write_DWORD(f, dw): 76 f.write(struct.pack('h', dw)) 77 78 79 def Write_INT(f, i): 80 f.write(struct.pack('i', i)) 81 82 83 def Write_UINT(f, i): 84 f.write(struct.pack('I', i)) 85 86 87 def Write_SHORT_STRING(f, s): 88 Write_BYTE(f, len(s)) 89 f.write(s) 90 91 92 def Write_LONG_STRING(f, s): 93 Write_INT(f, len(s)) 94 f.write(s) 95 96 97 def Write_REAL(f, r): 98 f.write(struct.pack('d', r)) 99 100 101 def Write_SINGLE(f, s): 102 f.write(struct.pack('f', s)) 103 104 105 def Write_LONGLONG(f, ll): 106 f.write(struct.pack('q', ll)) 107 108 109 dispatchInt = {P_BYTE: Read_SBYTE, 110 P_DWORD: Read_DWORD, 111 P_INT: Read_INT, 112 P_LONGLONG: Read_LONGLONG} 113 dispatchStr = {P_SHORT_STRING: Read_SHORT_STRING, 114 P_LONG_STRING: Read_LONG_STRING} 115 dispatchFlt = {P_REAL: Read_REAL, 116 P_SINGLE: Read_SINGLE} 117 118 def ReadInteger(f): 119 return dispatchInt[Read_BYTE(f)](f) 120 121 122 def ReadString(f): 123 return dispatchStr[Read_BYTE(f)](f).rstrip('\x00') 124 125 126 def ReadFloat(f): 127 return dispatchFlt[Read_BYTE(f)](f) 128 129 130 def ReadBoolean(f): 131 b = Read_BYTE(f) 132 if b ^ 8: 133 return True 134 return False 135 136 137 def ReadHandle(f): 138 Read_BYTE(f) 139 return struct.unpack('I', f.read(4))[0] 140 141 142 def WriteInteger(f, i): 143 if i >= -128 and i < 128: 144 Write_BYTE(f, P_BYTE) 145 Write_SBYTE(f, i) 146 elif i >= -32768 and i < 32768: 147 Write_BYTE(f, P_DWORD) 148 Write_DWORD(f, i) 149 elif i >= -2147483648 and i < 2147483648L: 150 Write_BYTE(f, P_INT) 151 Write_INT(f, i) 152 else: 153 Write_BYTE(f, P_LONGLONG) 154 Write_LONGLONG(f, i) 155 156 157 def WriteHandle(f, i): 158 Write_BYTE(f, P_INT) 159 Write_UINT(f, i) 160 161 162 def WriteString(f, s): 163 if len(s) < 256: 164 Write_BYTE(f, P_SHORT_STRING) 165 Write_SHORT_STRING(f, s) 166 else: 167 Write_BYTE(f, P_LONG_STRING) 168 Write_LONG_STRING(f, s) 169 170 171 def WriteFloat(f, fl): 172 Write_BYTE(f, P_REAL) 173 Write_REAL(f, fl) 174 return 175 s = struct.pack('f', fl) 176 if struct.unpack('f', s) == fl: 177 Write_BYTE(f, P_SINGLE) 178 f.write(s) 179 else: 180 Write_BYTE(f, P_REAL) 181 Write_REAL(f, fl) 182 183 184 def WriteBoolean(f, b): 185 Write_BYTE(f, 9 if b else 8) 186 187 188 def FixStr(s): 189 idx = s.find('\x00') 190 if idx < 0: 191 return s 192 return s[:idx] 193 194 195 def FiletimeToUnix(t): 196 return t * 1e-07 - epoch_diff 197 198 199 def UnixToFiletime(t): 200 return int((t + epoch_diff) * 10000000.0) 201 202 203 class Header(object): 204 205 @classmethod 206 def Read(self, f): 207 h = self() 208 h.label = ReadString(f) 209 h.description = ReadString(f) 210 h.created = ReadFloat(f) 211 h.modified = ReadFloat(f) 212 return h 213 214 def Write(self, f): 215 WriteString(f, self.label) 216 WriteString(f, self.description) 217 WriteFloat(f, self.created) 218 WriteFloat(f, self.modified) 219 220 def __repr__(self): 221 return '<%s %r %r %r %r>' % (self.__class__.__name__, 222 self.label, 223 self.description, 224 self.created, 225 self.modified) 226 227 228 class InitData(object): 229 230 def __init__(self, f): 231 if f is None: 232 return 233 self.computer = ReadString(f) 234 self.processname = ReadString(f) 235 self.pid = ReadInteger(f) 236 self.module = ReadHandle(f) 237 self.modulename = ReadString(f) 238 self.timestamp = Read_LONGLONG(f) 239 240 def Write(self, f): 241 WriteString(f, self.computer) 242 WriteString(f, self.processname) 243 WriteInteger(f, self.pid) 244 WriteHandle(f, self.module) 245 WriteString(f, self.modulename) 246 Write_LONGLONG(f, self.timestamp) 247 248 def __repr__(self): 249 return '<%s %r,%r,%r>' % (self.computer, 250 self.processname, 251 self.pid, 252 self.modulename) 253 254 255 class Channel(object): 256 257 def __init__(self, f): 258 if f is not None: 259 self.ReadChannelInfo(f) 260 261 infoFmt = 'qq33s33sii0q' 262 263 def ReadChannelInfo(self, f): 264 fmt = self.infoFmt 265 t = struct.unpack(fmt, f.read(struct.calcsize(fmt))) 266 self.facilityHash = t[0] 267 self.objectHash = t[1] 268 self.facility = FixStr(t[2]) 269 self.objectdesc = FixStr(t[3]) 270 self.suppressed = t[4] 271 self.flagSuppress = t[5] 272 273 def WriteChannelInfo(self, f): 274 f.write(struct.pack(self.infoFmt, self.facilityHash, self.objectHash, self.facility, self.objectdesc, self.suppressed, self.flagSuppress)) 275 276 def ReadHeader(self, f): 277 self.header = Header.Read(f) 278 279 def WriteHeader(self, f): 280 self.header.Write(f) 281 282 def __repr__(self): 283 return '<%s %r>' % (self.__class__.__name__, '::'.join((self.facility, self.objectdesc))) 284 285 def __str__(self): 286 return '::'.join((self.facility, self.objectdesc)) 287 288 289 class LBW(object): 290 291 def __init__(self, f = None): 292 if f: 293 self.Read(f) 294 295 def ReadDevices(self, f): 296 n = ReadInteger(f) 297 return [ self.ReadDevice(f) for d in xrange(n) ] 298 299 def ReadDevice(self, f): 300 device = {'header': Header.Read(f), 301 'devId': Read_LONGLONG(f), 302 'mapFileName': ReadString(f), 303 'pollInterval': ReadInteger(f), 304 'capacity': ReadInteger(f), 305 'channels': self.ReadChannels(f), 306 'start': ReadBoolean(f)} 307 return device 308 309 def WriteDevices(self, f, devices): 310 WriteInteger(f, len(devices)) 311 for d in devices: 312 self.WriteDevice(f, d) 313 314 def WriteDevice(self, f, d): 315 d['header'].Write(f) 316 Write_LONGLONG(f, d['devId']) 317 WriteString(f, d['mapFileName']) 318 WriteInteger(f, d['pollInterval']) 319 WriteInteger(f, d['capacity']) 320 self.WriteChannels(f, d['channels'], d['devId']) 321 WriteBoolean(f, d['start']) 322 323 def ReadChannels(self, f): 324 devId = Read_LONGLONG(f) 325 counter = ReadInteger(f) 326 numChannels = ReadInteger(f) 327 channels = [ Channel(f) for c in xrange(numChannels) ] 328 for channel in channels: 329 channel.ReadHeader(f) 330 initCounter = ReadInteger(f) 331 initData = {} 332 for i in xrange(initCounter): 333 sourceId = Read_INT(f) 334 initData[sourceId] = InitData(f) 335 336 channel.initData = initData 337 338 return channels 339 340 def WriteChannels(self, f, channels, devId): 341 Write_LONGLONG(f, devId) 342 nInits = sum([ len(chan.initData) for chan in channels ]) 343 WriteInteger(f, nInits) 344 WriteInteger(f, len(channels)) 345 for c in channels: 346 c.WriteChannelInfo(f) 347 348 for c in channels: 349 c.WriteHeader(f) 350 WriteInteger(f, len(c.initData)) 351 for id, data in c.initData.iteritems(): 352 Write_INT(f, id) 353 data.Write(f) 354 355 def ReadLogs(self, f): 356 self.createLogBase = ReadBoolean(f) 357 self.startPerformanceCounters = {} 358 result = [ self.ReadLogStorage(f) for i in xrange(ReadInteger(f)) ] 359 return result 360 361 def WriteLogs(self, f, logs): 362 WriteBoolean(f, self.createLogBase) 363 WriteInteger(f, len(logs)) 364 for l in logs: 365 self.WriteLogStorage(f, l) 366 367 def ReadLogStorage(self, f): 368 header = Header.Read(f) 369 logId = Read_LONGLONG(f) 370 initialCap = ReadInteger(f) 371 incrCap = ReadInteger(f) 372 clearOnNewPid = ReadBoolean(f) 373 filterChannels = ReadBoolean(f) 374 fileTime = Read_LONGLONG(f) 375 flags = [ReadInteger(f)] 376 while flags[-1] != -1: 377 flags.append(ReadInteger(f)) 378 379 filterFlags = ReadBoolean(f) 380 filterFlagsMask = ReadInteger(f) 381 keepLogs = ReadBoolean(f) 382 if keepLogs: 383 numberOfLogs = ReadInteger(f) 384 numberOfOverlap = ReadInteger(f) 385 logs = [ self.ReadLogLine(f) for i in xrange(numberOfLogs + numberOfOverlap) ] 386 else: 387 logs = [] 388 return {'header': header, 389 'logId': logId, 390 'initialCap': initialCap, 391 'incrCap': incrCap, 392 'clearOnNewPid': clearOnNewPid, 393 'filterChannels': filterChannels, 394 'fileTime': fileTime, 395 'filterFlags': filterFlags, 396 'filterFlagsMask': filterFlagsMask, 397 'keepLogs': keepLogs, 398 'flags': flags, 399 'logs': logs} 400 401 def WriteLogStorage(self, f, s): 402 s['header'].Write(f) 403 Write_LONGLONG(f, s['logId']) 404 WriteInteger(f, s['initialCap']) 405 WriteInteger(f, s['incrCap']) 406 WriteBoolean(f, s['clearOnNewPid']) 407 WriteBoolean(f, s['filterChannels']) 408 Write_LONGLONG(f, s['fileTime']) 409 for i in s['flags']: 410 WriteInteger(f, i) 411 412 WriteBoolean(f, s['filterFlags']) 413 WriteInteger(f, s['filterFlagsMask']) 414 WriteBoolean(f, s['keepLogs']) 415 if s['keepLogs']: 416 WriteInteger(f, len(s['logs'])) 417 WriteInteger(f, 0) 418 for line in s['logs']: 419 self.WriteLogLine(f, line) 420 421 def ReadLogLine(self, f): 422 return [Read_DWORD(f), 423 Read_INT(f), 424 Read_LONGLONG(f), 425 Read_INT(f), 426 Read_LONG_STRING(f).rstrip('\x00'), 427 Read_INT(f), 428 Read_INT(f)] 429 430 def WriteLogLine(self, f, line): 431 Write_DWORD(f, line[0]) 432 Write_INT(f, line[1]) 433 Write_LONGLONG(f, line[2]) 434 Write_INT(f, line[3]) 435 Write_LONG_STRING(f, line[4] + '\x00') 436 Write_INT(f, line[5]) 437 Write_INT(f, line[6]) 438 439 def BuildIndices(self): 440 self.channels = [None] 441 for dev in self.devices: 442 self.channels.extend(dev['channels']) 443 444 def Read(self, f): 445 self.version = ReadInteger(f) 446 self.header = Header.Read(f) 447 self.filename = ReadString(f) 448 self.devices = self.ReadDevices(f) 449 self.logStores = self.ReadLogs(f) 450 self.numLayouts = ReadInteger(f) 451 self.layoutHeaders = [] 452 for i in range(self.numLayouts): 453 self.layoutHeaders.append(Header.Read(f)) 454 455 self.lbwTrailer = f.read() 456 self.BuildIndices() 457 458 def Write(self, f): 459 WriteInteger(f, self.version) 460 self.header.Write(f) 461 WriteString(f, self.filename) 462 self.WriteDevices(f, self.devices) 463 self.WriteLogs(f, self.logStores) 464 WriteInteger(f, self.numLayouts) 465 for header in self.layoutHeaders: 466 header.Write(f) 467 468 if not hasattr(self, 'lbwTrailer'): 469 self.lbwTrailer = '\x02\x00\x02\xff' 470 f.write(self.lbwTrailer) 471 472 def Normalize(self): 473 used = {} 474 for store in self.logStores: 475 for line in store['logs']: 476 idx = line[0] 477 try: 478 used[idx].add(line[6]) 479 except KeyError: 480 used[idx] = set((line[6],)) 481 482 i, j = (1, 1) 483 remap = {} 484 for d in self.devices: 485 chns = d['channels'] 486 for ci in xrange(len(chns)): 487 if i in used: 488 remap[i] = j 489 j += 1 490 c = chns[ci] 491 ids = used[i] 492 c.initData = dict([ (k, c.initData[k]) for k in ids ]) 493 else: 494 chns[ci] = None 495 i += 1 496 497 for d in self.devices: 498 chns = d['channels'] 499 chns = [ c for c in chns if c ] 500 d['channels'] = chns 501 502 for store in self.logStores: 503 lines = store['logs'] 504 for line in lines: 505 line[0] = remap[line[0]] 506 507 self.devices = [ d for d in self.devices if d['channels'] ] 508 self.BuildIndices() 509 510 def ChunkLines(self, logStore): 511 timediff = 0.0001 512 i = iter(logStore['logs']) 513 chunk = [i.next()] 514 head = LogLine(chunk[0], self.channels) 515 for l in i: 516 this = LogLine(l, self.channels) 517 if this.severity != head.severity or this.channel != head.channel or this.threadId != head.threadId or this.timestamp - head.timestamp >= timediff: 518 yield chunk 519 chunk = [l] 520 head = this 521 else: 522 chunk.append(l) 523 524 yield chunk 525 526 def Filter(self, chunk = False, pid = None, regex = None): 527 if regex is not None: 528 regex = re.compile(regex) 529 for store in self.logStores: 530 if chunk: 531 chunks = self.ChunkLines(store) 532 else: 533 chunks = [ [l] for l in store['logs'] ] 534 535 def acceptchunk(c): 536 for line in c: 537 if pid is not None and line[5] == pid: 538 return True 539 if regex and regex.search(line[4]): 540 return True 541 542 return False 543 544 chunks = [ c for c in chunks if acceptchunk(c) ] 545 lines = [] 546 for c in chunks: 547 lines.extend(c) 548 549 store['logs'] = lines 550 551 self.Normalize() 552 553 def Lines(self, logStore = 0): 554 s = self.logStores[0] 555 for line in s['logs']: 556 yield LogLine(line, self.channels) 557 558 def SetupNewWorkspace(self): 559 self._SetVersion(9) 560 self._SetHeader('Workspace', '') 561 self._SetFileName('defaultFileName') 562 self.devices = [] 563 self.AddDevice('defaultDevice', '', 1) 564 self.createLogBase = False 565 self.logStores = [] 566 self.startPerformanceCounters = {} 567 self.AddLogStorage('Default storage', '') 568 self.numLayouts = 0 569 self.layoutHeaders = [] 570 self.AddLayout('Default layout', '') 571 572 def _CreateNewHeader(self, label, description): 573 newHeader = Header() 574 newHeader.label = label 575 newHeader.description = description 576 newHeader.created = self._GetCurrentTimeInDelphiFormat() 577 newHeader.modified = self._GetCurrentTimeInDelphiFormat() 578 return newHeader 579 580 def _SetVersion(self, version): 581 self.version = version 582 583 def _SetHeader(self, label, description): 584 self.header = self._CreateNewHeader(label, description) 585 586 def _SetFileName(self, fileName): 587 self.filename = fileName 588 589 def AddDevice(self, deviceLabel, deviceDescription, deviceID, pollInterval = 500, capacity = 5000): 590 deviceHeader = self._CreateNewHeader(deviceLabel, deviceDescription) 591 device = {'header': deviceHeader, 592 'devId': deviceID, 593 'mapFileName': deviceLabel, 594 'pollInterval': pollInterval, 595 'capacity': capacity, 596 'channels': [], 597 'start': True} 598 self.devices.append(device) 599 600 def AddLogStorage(self, label, description, initialCapacity = 1000, incrementalCapacity = 1000, clearOnNewPid = False, filterFlags = False, filterFlagsMask = 0): 601 storageHeader = self._CreateNewHeader(label, description) 602 devID = self.devices[0]['devId'] 603 logStorage = {'header': storageHeader, 604 'logId': devID, 605 'initialCap': initialCapacity, 606 'incrCap': incrementalCapacity, 607 'clearOnNewPid': clearOnNewPid, 608 'filterChannels': False, 609 'fileTime': 0, 610 'filterFlags': filterFlags, 611 'filterFlagsMask': filterFlagsMask, 612 'keepLogs': True, 613 'flags': [-1], 614 'logs': []} 615 self.logStores.append(logStorage) 616 617 def AddLayout(self, label, description): 618 self.numLayouts += 1 619 layoutHeader = self._CreateNewHeader(label, description) 620 self.layoutHeaders.append(layoutHeader) 621 622 def _GetCurrentTimeInDelphiFormat(self): 623 curTime = datetime.datetime.utcnow() - DELPHI_DATE_TIME_EPOCH 624 return curTime.days + curTime.seconds / 86400.0 625 626 def AddChannel(self, facilityHash, objectHash, facilityName, objectDescription, sourceID, computerName, processName, processID, module, moduleName, timestamp): 627 channel = None 628 for chan in self.devices[0]['channels']: 629 if chan.facilityHash == facilityHash and chan.objectHash == objectHash and chan.facility == facilityName and chan.objectdesc == objectDescription: 630 channel = chan 631 break 632 633 if channel is None: 634 channel = Channel(None) 635 channel.facilityHash = facilityHash 636 channel.objectHash = objectHash 637 channel.facility = facilityName 638 channel.objectdesc = objectDescription 639 channel.suppressed = 0 640 channel.flagSuppress = 0 641 channel.header = self._CreateNewHeader(objectDescription, '') 642 channel.initData = {} 643 self.devices[0]['channels'].append(channel) 644 channelInitData = InitData(None) 645 channelInitData.computer = computerName 646 channelInitData.processname = processName 647 channelInitData.pid = processID 648 channelInitData.module = module 649 channelInitData.modulename = moduleName 650 channelInitData.timestamp = timestamp 651 channel.initData[sourceID] = channelInitData 652 self.BuildIndices() 653 654 def AddLogLine(self, channelIndex, threadID, performanceCounter, flag, message, processID, sourceID): 655 for idx, store in enumerate(self.logStores): 656 if not store['filterFlags'] or flag & store['filterFlagsMask']: 657 if len(store['logs']) == 0: 658 store['fileTime'] = UnixToFiletime(time.time()) 659 if idx not in self.startPerformanceCounters or self.startPerformanceCounters[idx] == 0: 660 self.startPerformanceCounters[idx] = performanceCounter 661 timestamp = long((performanceCounter - self.startPerformanceCounters[idx]) * 10000000.0 / COUNTER_FREQUENCY) 662 timestamp += store['fileTime'] 663 logLine = [channelIndex, 664 threadID, 665 timestamp, 666 flag, 667 message, 668 processID, 669 sourceID] 670 store['logs'].append(logLine) 671 672 673 class LogLine(object): 674 __slots__ = ['channel', 675 'threadId', 676 '_timestamp', 677 'severity', 678 'msg', 679 'pid', 680 'initData'] 681 682 def __init__(self, t, channels): 683 self.channel = channels[t[0]] 684 self.threadId = t[1] 685 self._timestamp = t[2] 686 self.severity = t[3] 687 self.msg = t[4] 688 self.pid = t[5] 689 self.initData = self.channel.initData[t[6]] 690 691 @property 692 def message(self): 693 try: 694 return self.msg.decode('utf-8') 695 except UnicodeDecodeError: 696 try: 697 return self.msg.decode('cp1252') 698 except UnicodeDecodeError: 699 return self.msg 700 701 @property 702 def timestamp(self): 703 return FiletimeToUnix(self._timestamp) 704 705 @property 706 def facility(self): 707 return self.channel.facility 708 709 @property 710 def object(self): 711 return self.channel.objectdesc 712 713 @property 714 def computer(self): 715 return self.initData.computer 716 717 @property 718 def process(self): 719 return self.initData.processname 720 721 @property 722 def module(self): 723 return self.initData.modulename 724 725 @property 726 def severitystr(self): 727 return severity_name[self.severity] 728 729 def ShortTime(self): 730 ts = self.timestamp 731 ms = int(ts * 1000.0) % 1000 732 return time.strftime('%H:%M:%S', time.gmtime(ts)) + '.%.3i' % ms 733 734 def LongTime(self): 735 ts = self.timestamp 736 ms = int(ts * 1000.0) % 1000 737 return time.strftime('%Y.%m.%d::%H:%M:%S', time.gmtime(ts)) + '.%.3i' % ms 738 739 def __repr__(self): 740 '<%s %r>' % (self.__class__.__name__, self.message[:20]) 741 742 def __str__(self): 743 ts = self.timestamp 744 ms = int(ts * 1000.0) % 1000 745 return '%s:%s:%s:%r' % (self.ShortTime(), 746 self.facility, 747 self.object, 748 self.message) 749 750 751 def ChunkLines(lines): 752 lineIter = iter(lines) 753 chunk = [lineIter.next()] 754 last = chunk[0] 755 regex = re.compile('\\n- *') 756 timediff = 0.0001 757 for l in lineIter: 758 if l.severity != last.severity or l.channel != last.channel or l.threadId != last.threadId or l.timestamp - last.timestamp >= timediff: 759 last.msg = '\n'.join([ line.msg for line in chunk ]) 760 last.msg = regex.sub('', last.msg) 761 yield last 762 chunk = [l] 763 last = l 764 else: 765 chunk.append(l) 766 767 768 def Filter(lines, severity = 0, pid = 0, text = None): 769 lineIter = iter(lines) 770 if text: 771 import re 772 re = re.compile(text) 773 else: 774 re = None 775 for l in lineIter: 776 if re and not re.search(l.message): 777 continue 778 if severity and not l.severity & severity: 779 continue 780 if pid and pid != l.pid: 781 continue 782 yield l 783 784 785 def FilterFiles(files, severity = None, text = None): 786 sev = dict(info=1, warn=2, error=4, fatal=8) 787 sev = sev[severity] if severity else 0 788 for fn in files: 789 if not hasattr(fn, 'read'): 790 f = file(fn, 'rb') 791 else: 792 f = fn 793 logs = LBW(f) 794 for l in Filter(ChunkLines(logs.Lines()), severity=sev, text=text): 795 print(l.ShortTime(), '%6i' % l.pid, '%5s' % l.severitystr, '%20s' % l.channel, repr(l.message)) 796 797 798 def FilterExceptions(files): 799 regex = re.compile('\\b.+Error: .+') 800 for fn in files: 801 if not hasattr(fn, 'read'): 802 f = file(fn, 'rb') 803 else: 804 f = fn 805 logs = LBW(f) 806 for l in ChunkLines(logs.Lines()): 807 msg = l.message 808 m = regex.search(msg) 809 if m: 810 print(l.ShortTime(), '%6i' % l.pid, '%5s' % l.severitystr, '%20s' % l.channel, repr(m.group(0)), repr(msg)) 811 812 813 def FilterFile(infile, outfile, chunk = False, pid = None, regex = None): 814 logs = LBW(open(infile, 'rb')) 815 logs.Filter(chunk=chunk, pid=pid, regex=regex) 816 logs.Write(open(outfile, 'wb')) 817 818 819 def main(args = None): 820 import glob 821 if args is None: 822 args = sys.argv[1:] 823 if not args or args[0] not in ('-l', '-s', '-t', '-e', '-f'): 824 Usage() 825 if args[0] == '-l': 826 if len(args) == 1: 827 Usage() 828 for f in args[1:]: 829 FilterFiles(glob.glob(f)) 830 831 if args[0] == '-s': 832 if len(args) < 3: 833 Usage() 834 severity = args[1] 835 for f in args[2:]: 836 FilterFiles(glob.glob(f), severity=severity) 837 838 if args[0] == '-t': 839 if len(args) < 3: 840 Usage() 841 text = args[1] 842 for f in args[2:]: 843 FilterFiles(glob.glob(f), text=text) 844 845 if args[0] == '-e': 846 if len(args) < 2: 847 Usage() 848 for f in args[1:]: 849 FilterExceptions(glob.glob(f)) 850 851 if args[0] == '-f': 852 args.pop(0) 853 try: 854 outfile = args.pop(-1) 855 infile = args.pop(-1) 856 pid = None 857 regex = None 858 chunk = False 859 while args: 860 if args[0] == '-p': 861 args.pop(0) 862 pid = int(args.pop(0)) 863 elif args[0] == '-c': 864 args.pop(0) 865 chunk = True 866 elif args[0] == '-r': 867 args.pop(0) 868 regex = args.pop(0) 869 else: 870 Usage() 871 872 except IndexError: 873 Usage() 874 875 FilterFile(infile, outfile, pid=pid, chunk=chunk, regex=regex) 876 877 878 def Usage(): 879 import textwrap 880 print(textwrap.dedent(' Usage:\n lbw.py -l files ... \n lbw.py -s severity files \n lbw.py -e files # Exceptions\n lbw.py -f [-c] [-r regex][-p pid] infile outfile #filter file\n ')) 881 sys.exit(1)