htmlutils.py (20978B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\scriber\htmlutils.py 2 from __future__ import print_function 3 import re 4 import HTMLParser 5 import datetime 6 import datetimeutils 7 import typeutils 8 from scriber import const 9 from inventorycommon import const as iconst 10 from functools import reduce 11 12 class TolerantParser(HTMLParser.HTMLParser): 13 ON_ERROR_THROW = 0 14 ON_ERROR_REPLACE = 1 15 ON_ERROR_SKIP = 2 16 17 def __init__(self): 18 HTMLParser.HTMLParser.__init__(self) 19 self.on_error = self.ON_ERROR_REPLACE 20 self.parse_errors = [] 21 22 def on_error_replace_with(self): 23 return '<' 24 25 def parse_starttag(self, i): 26 try: 27 return HTMLParser.HTMLParser.parse_starttag(self, i) 28 except HTMLParser.HTMLParseError as e: 29 self.parse_errors.append(e) 30 if self.on_error == self.ON_ERROR_REPLACE: 31 self.handle_data(self.on_error_replace_with()) 32 return i + 1 33 if self.on_error == self.ON_ERROR_SKIP: 34 return i + 1 35 raise 36 37 def parse_endtag(self, i): 38 try: 39 return HTMLParser.HTMLParser.parse_endtag(self, i) 40 except HTMLParser.HTMLParseError as e: 41 self.parse_errors.append(e) 42 if self.on_error == self.ON_ERROR_REPLACE: 43 self.handle_data(self.on_error_replace_with()) 44 return i + 1 45 if self.on_error == self.ON_ERROR_SKIP: 46 return i + 1 47 raise 48 49 def unknown_decl(self, data): 50 pass 51 52 def goahead(self, end): 53 rawdata = self.rawdata 54 i = 0 55 n = len(rawdata) 56 while i < n: 57 match = self.interesting.search(rawdata, i) 58 if match: 59 j = match.start() 60 else: 61 j = n 62 if i < j: 63 self.handle_data(rawdata[i:j]) 64 i = self.updatepos(i, j) 65 if i == n: 66 break 67 startswith = rawdata.startswith 68 if startswith('<', i): 69 if HTMLParser.starttagopen.match(rawdata, i): 70 k = self.parse_starttag(i) 71 elif startswith('</', i): 72 k = self.parse_endtag(i) 73 elif startswith('<!--', i): 74 k = self.parse_comment(i) 75 elif startswith('<?', i): 76 k = self.parse_pi(i) 77 elif startswith('<!', i): 78 k = self.parse_declaration(i) 79 elif i + 1 < n: 80 self.handle_data('<') 81 k = i + 1 82 else: 83 break 84 if k < 0: 85 if end: 86 self.error('EOF in middle of construct') 87 break 88 i = self.updatepos(i, k) 89 elif startswith('&#', i): 90 match = HTMLParser.charref.match(rawdata, i) 91 if match: 92 name = match.group()[2:-1] 93 k = match.end() 94 if not startswith(';', k - 1): 95 self.handle_data('&#' + name) 96 k = k - 1 97 else: 98 self.handle_charref(name) 99 i = self.updatepos(i, k) 100 continue 101 else: 102 if ';' in rawdata[i:]: 103 self.handle_data(rawdata[0:2]) 104 i = self.updatepos(i, 2) 105 break 106 elif startswith('&', i): 107 match = HTMLParser.entityref.match(rawdata, i) 108 if match: 109 name = match.group(1) 110 k = match.end() 111 if not startswith(';', k - 1): 112 self.handle_data('&' + name) 113 k = k - 1 114 else: 115 self.handle_entityref(name) 116 i = self.updatepos(i, k) 117 continue 118 match = HTMLParser.incomplete.match(rawdata, i) 119 if match: 120 self.handle_data(rawdata[i:]) 121 if end and match.group() == rawdata[i:]: 122 self.error('EOF in middle of entity or char ref') 123 break 124 elif i + 1 < n: 125 self.handle_data('&') 126 i = self.updatepos(i, i + 1) 127 else: 128 self.handle_data('&') 129 break 130 131 if end and i < n: 132 self.handle_data(rawdata[i:n]) 133 i = self.updatepos(i, n) 134 self.rawdata = rawdata[i:] 135 136 137 class TagStripper(TolerantParser): 138 139 def __init__(self, tag_list = tuple(), preserve_content_list = tuple()): 140 TolerantParser.__init__(self) 141 self._stripped = [] 142 self.tag_list = tag_list 143 self.preserve_content_list = preserve_content_list 144 self.ignore_content_until = None 145 146 def handle_data(self, data): 147 if not self.ignore_content_until: 148 self._stripped.append(data) 149 150 def handle_starttag(self, tag, attrs): 151 if tag in self.tag_list: 152 if tag not in self.preserve_content_list: 153 self.ignore_content_until = tag 154 else: 155 attributes = '' 156 if attrs: 157 attributes = ' %s' % attr_str(attrs) 158 self._stripped.append('<%s%s>' % (tag, attributes)) 159 160 def handle_endtag(self, tag): 161 if self.ignore_content_until == tag: 162 self.ignore_content_until = None 163 if tag not in self.tag_list: 164 self._stripped.append('</%s>' % tag) 165 166 def handle_entityref(self, name): 167 if not self.ignore_content_until: 168 self._stripped.append('&%s;' % name) 169 170 def handle_charref(self, name): 171 if not self.ignore_content_until: 172 self._stripped.append('&#%s;' % name) 173 174 def get_stripped(self): 175 return ''.join(self._stripped) 176 177 def flush_stripped(self): 178 text = self.get_stripped() 179 self._stripped = [] 180 return text 181 182 183 class HTMLStripper(TolerantParser): 184 HTML_WHITESPACE = '[ \\t\\r\\n]+' 185 HTML_BREAKS = '( \\n|\\n )' 186 HTML_NON_CONTENT_TAGS = ['head', 187 'script', 188 'style', 189 'embed', 190 'frameset', 191 'object', 192 'iframe', 193 'source', 194 'track', 195 'video', 196 'audio', 197 'canvas', 198 'meta'] 199 200 def __init__(self, preserve_links = True, preserve_images = False, decode_chars = True, error_tolerance = True): 201 TolerantParser.__init__(self) 202 self._text = [] 203 self.preserve_links = preserve_links 204 self.preserve_images = preserve_images 205 self.decode_chars = decode_chars 206 self.error_tolerance = error_tolerance 207 self.link = '' 208 self.ignore_content_until = None 209 210 def handle_data(self, data): 211 if not self.ignore_content_until: 212 self._text.append(data) 213 214 def handle_starttag(self, tag, attrs): 215 if tag in self.HTML_NON_CONTENT_TAGS: 216 self.ignore_content_until = tag 217 if tag == 'a': 218 if self.preserve_links and attrs: 219 attrs = dict(attrs) 220 self.link = attrs.get('href', '') 221 if tag == 'img': 222 if self.preserve_images and attrs: 223 attrs = dict(attrs) 224 self._text.append('[%s]' % attrs.get('src', '')) 225 elif tag == 'br': 226 self._add_linefeed() 227 228 def handle_endtag(self, tag): 229 if self.ignore_content_until == tag: 230 self.ignore_content_until = None 231 if tag in ('p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'): 232 self._add_linefeed(2) 233 elif tag == 'a' and self.preserve_links and self.link: 234 self._text.append(' [%s]' % self.link) 235 self.link = '' 236 237 def handle_entityref(self, name): 238 if not self.ignore_content_until: 239 code = '&%s;' % name 240 if self.decode_chars: 241 code = self.unescape(code) 242 self._text.append(code) 243 244 def handle_charref(self, name): 245 if not self.ignore_content_until: 246 code = '&#%s;' % name 247 if self.decode_chars: 248 code = self.unescape(code) 249 self._text.append(code) 250 251 def on_error_replace_with(self): 252 if self.decode_chars: 253 return '[' 254 else: 255 return '<' 256 257 def _add_linefeed(self, count = 1): 258 self._text.append('{{LF}}' * count) 259 260 def get_text(self): 261 text = ''.join(self._text) 262 text = re.sub(self.HTML_WHITESPACE, ' ', text) 263 text = text.replace('{{LF}}', '\n') 264 text = re.sub(self.HTML_BREAKS, '\n', text) 265 return text.strip() 266 267 def flush_text(self): 268 text = self.get_text() 269 self._text = [] 270 return text 271 272 273 class AbstractCleaner(object): 274 275 def __init__(self, markup): 276 self.markup = markup 277 self.ok = True 278 self.clean_markup = '' 279 self.missing_tags = None 280 self.break_idx = -1 281 self.fixed_markup = None 282 self.html = '' 283 self.length = 0 284 285 def cleanup(self): 286 if self.markup and isinstance(self.markup, (str, unicode)): 287 self.length = len(self.markup) 288 self.ok, self.missing_tags, self.break_idx = validate_evemarkup(self.markup) 289 self.clean_markup = remove_loc_tags(self.markup) 290 self.clean_markup = remove_empty_formatting_tags(self.clean_markup) 291 if not self.ok: 292 motd_ok2, missing_tags2, break_idx2 = validate_evemarkup(self.clean_markup) 293 if not motd_ok2: 294 self.fixed_markup = fix_broken_markup(self.clean_markup, missing_tags2, break_idx2) 295 self.html = evemarkup_to_html(self.fixed_markup) 296 else: 297 self.html = evemarkup_to_html(self.clean_markup) 298 else: 299 self.html = evemarkup_to_html(self.clean_markup) 300 301 302 class MotdCleaner(AbstractCleaner): 303 304 def __init__(self, live_motd, channel): 305 super(MotdCleaner, self).__init__(live_motd) 306 self.channel = channel 307 self.db_motd_length = 0 308 self.has_loc = None 309 310 def cleanup(self): 311 if self.markup: 312 self.has_loc = '<loc>' in self.markup 313 else: 314 self.has_loc = False 315 if self.channel.motd: 316 self.db_motd_length = len(self.channel.motd) 317 super(MotdCleaner, self).cleanup() 318 319 320 class EveMailCleaner(AbstractCleaner): 321 322 def __init__(self, body): 323 super(EveMailCleaner, self).__init__(body) 324 325 326 def strip_html(html_string, preserve_links = True, preserve_images = False, decode_chars = True): 327 stripper = HTMLStripper(preserve_links, preserve_images, decode_chars) 328 stripper.feed(html_string) 329 return stripper.get_text() 330 331 332 def newline_to_html(string): 333 return '<p>%s</p>' % reduce(lambda h, n: h.replace(*n), (('\r', ''), ('\n\n', '</p><p>'), ('\n', '<br />')), string) 334 335 336 def sanitize(html_string, keep_amp = False): 337 if not keep_amp: 338 html_string = html_string.replace('&', '&') 339 return reduce(lambda string, params: string.replace(*params), (('<', '<'), ('>', '>')), html_string) 340 341 342 def unsanitize(string): 343 return reduce(lambda string, params: string.replace(*params), (('>', '>'), ('<', '<'), ('&', '&')), string) 344 345 346 def esc_email_tags(html, replacer = const.EMAIL_TAG_REPLACE_HTML): 347 return const.EMAIL_TAG_GRABBER.sub(replacer, html) 348 349 350 def strip_bad_tags(html_string): 351 stripper = TagStripper(HTMLStripper.HTML_NON_CONTENT_TAGS) 352 stripper.feed(html_string) 353 return stripper.get_stripped() 354 355 356 def attr_str(attribute_list, xhtml_strict = False): 357 buff = [] 358 if isinstance(attribute_list, dict): 359 attribute_list = attribute_list.items() 360 for attr in attribute_list: 361 if len(attr) > 1: 362 buff.append('%s="%s"' % (attr[0], attr[1])) 363 elif xhtml_strict: 364 buff.append('%s="%s"' % (attr[0], attr[0])) 365 else: 366 buff.append('%s' % attr[0]) 367 368 return ' '.join(buff) 369 370 371 def parse_user_notes(raw_notes, note_date = datetime.datetime.now()): 372 raw_notes = sanitize(raw_notes, keep_amp=True) 373 raw_notes = parse_markdown_link(raw_notes) 374 note_date = datetimeutils.any_to_datetime(note_date) 375 i = 0 376 parsed_notes = [] 377 for m in const.NOTE_LINK_A_HREF_MATCHER.finditer(raw_notes): 378 parsed_notes.append(parse_note_auto_link(raw_notes[i:m.start()], note_date)) 379 parsed_notes.append(m.group(0)) 380 i = m.end() 381 382 parsed_notes.append(parse_note_auto_link(raw_notes[i:], note_date)) 383 return ''.join(parsed_notes).replace('\n', '<br>') 384 385 386 def parse_markdown_link(raw_text): 387 return re.sub(const.NOTE_LINK_MARKDOWN_PATTERN, const.NOTE_LINK_MARKDOWN_REPLACE, raw_text) 388 389 390 def parse_note_auto_link(raw_text, note_date): 391 i = 0 392 parsed_text = [] 393 for m in const.NOTE_LINK_AUTO_LINKER.finditer(raw_text): 394 parsed_text.append(raw_text[i:m.start()]) 395 parsed_text.append(const.NOTE_LINK_AUTO_LINK.format(url=_get_ticket_url(m.group(3), note_date), text='%s%s%s' % (m.group(1), m.group(2), m.group(3)))) 396 i = m.end() 397 398 parsed_text.append(raw_text[i:]) 399 return ''.join(parsed_text) 400 401 402 def _get_ticket_url(ticket_id, note_date): 403 ticket_id = typeutils.int_eval(ticket_id) 404 if ticket_id > const.OLD_TICKET_ID_THRESHOLD and note_date.year < 2016: 405 return const.OLD_TICKET_URL.format(ticketID=ticket_id) 406 return const.TICKET_URL.format(ticketID=ticket_id) 407 408 409 def strip_extra_amp(string): 410 return re.sub(const.EXTRA_AMP_MATCHER, const.EXTRA_AMP_REPLACER, string) 411 412 413 def esp_item_link(item): 414 if item.item_type.category_id == iconst.categoryOwner: 415 if item.item_type.group_id == iconst.groupCharacter: 416 return '/gm/character.py?action=Character&characterID=%d' % item.item_id 417 if item.item_type.group_id == iconst.groupCorporation: 418 return '/gm/corporation.py?action=Corporation&corporationID=%d' % item.item_id 419 if item.item_type.group_id == iconst.groupAlliance: 420 return '/gm/alliance.py?action=Alliance&allianceID=%d' % item.item_id 421 if item.item_type.group_id == iconst.groupFaction: 422 return '/gm/faction.py?action=Faction&factionID=%d' % item.item_id 423 elif item.item_type.category_id == iconst.categoryCelestial: 424 if item.item_type.group_id == iconst.groupRegion: 425 return '/gd/universe.py?action=Region®ionID=%d' % item.item_id 426 if item.item_type.group_id == iconst.groupConstellation: 427 return '/gd/universe.py?action=Constellation&constellationID=%s' % item.item_id 428 if item.item_type.group_id == iconst.groupSolarSystem: 429 return '/gd/universe.py?action=System&systemID=%d' % item.item_id 430 if item.item_type.group_id == iconst.groupAsteroidBelt: 431 return '/gd/universe.py?action=AsteroidBelt&asteroidBeltID=%d' % item.item_id 432 if item.item_type.group_id in [iconst.groupPlanet, iconst.groupMoon]: 433 return '/gd/universe.py?action=Celestial&celestialID=%d' % item.item_id 434 elif item.item_type.category_id == iconst.categoryStation: 435 if item.item_type.group_id == iconst.groupStation: 436 return '/gm/stations.py?action=Station&stationID=%d' % item.item_id 437 else: 438 if item.item_type.category_id == iconst.categorySovereigntyStructure: 439 return '/gm/starbase.py?action=Starbase&towerID=%d' % item.item_id 440 if item.item_type.category_id == iconst.categoryShip: 441 return '/gm/character.py?action=Ship&characterID=%d&shipID=%d' % (item.owner_id, item.item_id) 442 return '/gm/inventory.py?action=Item&itemID=%d' % item.item_id 443 444 445 def validate_evemarkup(markup): 446 markup = markup.lower() 447 idx = -1 448 if '<' in markup: 449 idx = markup.rindex('<') 450 last_part = markup[idx:] 451 if '>' in last_part: 452 idx = -1 453 if idx > -1: 454 markup = markup[:idx] 455 start_less_markup = [] 456 tag_starts_found = [] 457 p = 0 458 for match in re.finditer('<(\\w+)[^>]*>', markup): 459 start_less_markup.append(markup[p:match.start()]) 460 p = match.end() 461 tag = match.group(1) 462 if tag != 'br': 463 tag_starts_found.append(tag) 464 465 start_less_markup.append(markup[p:]) 466 markup_no_start = ''.join(start_less_markup) 467 for match in re.finditer('</(\\w+)>', markup_no_start): 468 tag = match.group(1) 469 if tag != 'br': 470 if tag in tag_starts_found: 471 tag_starts_found.remove(tag) 472 473 return (idx == -1 and not tag_starts_found, tag_starts_found, idx) 474 475 476 def fix_broken_markup(markup, close_tags_list = None, index_of_broken_tag = -1): 477 if index_of_broken_tag > -1: 478 markup = markup[:index_of_broken_tag] 479 if close_tags_list: 480 append_tags = [ '</%s>' % t for t in close_tags_list ] 481 markup = '%s%s' % (markup, ''.join(append_tags)) 482 return markup 483 484 485 def remove_empty_formatting_tags(markup): 486 tag_list = ('fontsize', 'color', 'i', 'u', 'b', 'url') 487 for tag in tag_list: 488 fixed_markup = [] 489 p = 0 490 for match in re.finditer('(?i)<%s[^>]*></%s>' % (tag, tag), markup): 491 fixed_markup.append(markup[p:match.start()]) 492 p = match.end() 493 494 fixed_markup.append(markup[p:]) 495 markup = ''.join(fixed_markup) 496 497 return markup 498 499 500 def remove_tags(markup, *args): 501 tag_list = [] 502 if args: 503 for t in args: 504 if isinstance(t, (list, tuple)): 505 tag_list.extend(t) 506 else: 507 tag_list.append(t) 508 509 for tag in tag_list: 510 markup = re.sub('(?i)<%s[^>]*>(.*?)</%s>' % (tag, tag), '\\1', markup) 511 512 return markup 513 514 515 def remove_loc_tags(markup): 516 return markup.replace('<loc>', '').replace('</loc>', '') 517 518 519 def add_loc_tags(markup): 520 return re.sub(const.SHOW_INFO_PATTERN, const.SHOW_INFO_REPLACE, markup) 521 522 523 def evemarkup_to_html(markup): 524 markup = remove_loc_tags(markup) 525 markup = re.sub(const.EVEML_SHOWINFO_PATTERN, const.EVEML_SHOWINFO_REPLACE, markup) 526 markup = re.sub(const.EVEML_FONT_PATTERN, const.EVEML_FONT_REPLACE, markup) 527 markup = re.sub(const.EVEML_FONTSIZE_PATTERN, const.EVEML_FONTSIZE_REPLACE, markup) 528 markup = re.sub(const.EVEML_COLOR_PATTERN, const.EVEML_COLOR_REPLACE, markup) 529 markup = re.sub(const.EVEML_URL_PATTERN, const.EVEML_URL_REPLACE, markup) 530 return markup 531 532 533 def markup_syntax_html(markup): 534 pass 535 536 537 if __name__ == '__main__': 538 s1 = "<fontsize=36>Stuff</fontsize><br><fontsize=36></fontsize><br><fontsize=14>* <b>Here's</b> a <loc><url=http://mbl.is>thin</loc>g</url><br>* Here's another <loc><url=showinfo:1373//90000012>thin</loc>g</url><br>* More <loc><url=showinfo:5//30000142>stuf</loc>f</url><br>* And <i>some</i> <loc><url=showinfo:54//60014839>stuff</url></loc> to <color=0xff00ff00>boot</color><br>* Wny > no <loc><url=showinfo:2//98000002>type</url></loc>?!?<br>* Let's try <loc><color=0xffff0000>type</loc> again!</fontsize></color>" 539 s2 = "<fontsize=36>Stuff</fontsize><br><br><fontsize=14>* <b>Here's</b> a <loc><url=http://mbl.is>thin</loc>g</url><br>* Here's another <loc><url=showinfo:1373//90000012>thin</loc>g</url><br>* More <loc><url=showinfo:5//30000142>stuf</loc>f</url><br>* And <i>some</i> <loc><url=showinfo:54//60014839>stuff</url></loc> to <color=0xff00ff00>boot</color><br>* Wny > no <loc><url=showinfo:2//98000002>type</url></loc>?!?<br>* Let's try <loc><color=0xffff0000>type</loc> again!" 540 s3 = "<fontsize=36>Stuff</fontsize><br><br><b>Here's</b> a <loc" 541 s4 = "<fontsize=36>Stuff<br><br><b>Here's</b> a <loc" 542 nl1 = remove_loc_tags(s1) 543 nl2 = remove_loc_tags(s2) 544 nl3 = remove_loc_tags(s3) 545 nl4 = remove_loc_tags(s4) 546 print('validate_evemarkup') 547 ok1, tl1, ix1 = validate_evemarkup(nl1) 548 print(ok1, tl1, ix1) 549 ok2, tl2, ix2 = validate_evemarkup(nl2) 550 print(ok2, tl2, ix2) 551 ok3, tl3, ix3 = validate_evemarkup(nl3) 552 print(ok3, tl3, ix3) 553 ok4, tl4, ix4 = validate_evemarkup(nl4) 554 print(ok4, tl4, ix4) 555 print('fix_broken_markup') 556 f1 = fix_broken_markup(nl1, tl1, ix1) 557 print(f1) 558 f2 = fix_broken_markup(nl3, tl3, ix3) 559 print(f2) 560 f3 = fix_broken_markup(nl2, tl2, ix2) 561 print(f3) 562 f4 = fix_broken_markup(nl4, tl4, ix4) 563 print(f4) 564 print('remove_empty_formatting_tags') 565 ff1 = remove_empty_formatting_tags(f1) 566 print(ff1) 567 ff2 = remove_empty_formatting_tags(f2) 568 print(ff2) 569 ff3 = remove_empty_formatting_tags(f3) 570 print(ff3) 571 ff4 = remove_empty_formatting_tags(f4) 572 print(ff4) 573 print('add_loc_tags') 574 fi1 = add_loc_tags(ff1) 575 print(fi1) 576 fi2 = add_loc_tags(ff2) 577 print(fi2) 578 fi3 = add_loc_tags(ff3) 579 print(fi3) 580 fi4 = add_loc_tags(ff4) 581 print(fi4) 582 print('----') 583 print(remove_tags(fi1, 'fontsize', 'color'))