NovaBryan/EVE_Dump
Log | Files | 📦 下载最新源码 (.zip) | Refs | README

htmlutils.py (20978B)


      1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\packages\scriber\htmlutils.py
      2 from __future__ import print_function
      3 import re
      4 import HTMLParser
      5 import datetime
      6 import datetimeutils
      7 import typeutils
      8 from scriber import const
      9 from inventorycommon import const as iconst
     10 from functools import reduce
     11 
     12 class TolerantParser(HTMLParser.HTMLParser):
     13     ON_ERROR_THROW = 0
     14     ON_ERROR_REPLACE = 1
     15     ON_ERROR_SKIP = 2
     16 
     17     def __init__(self):
     18         HTMLParser.HTMLParser.__init__(self)
     19         self.on_error = self.ON_ERROR_REPLACE
     20         self.parse_errors = []
     21 
     22     def on_error_replace_with(self):
     23         return '<'
     24 
     25     def parse_starttag(self, i):
     26         try:
     27             return HTMLParser.HTMLParser.parse_starttag(self, i)
     28         except HTMLParser.HTMLParseError as e:
     29             self.parse_errors.append(e)
     30             if self.on_error == self.ON_ERROR_REPLACE:
     31                 self.handle_data(self.on_error_replace_with())
     32                 return i + 1
     33             if self.on_error == self.ON_ERROR_SKIP:
     34                 return i + 1
     35             raise
     36 
     37     def parse_endtag(self, i):
     38         try:
     39             return HTMLParser.HTMLParser.parse_endtag(self, i)
     40         except HTMLParser.HTMLParseError as e:
     41             self.parse_errors.append(e)
     42             if self.on_error == self.ON_ERROR_REPLACE:
     43                 self.handle_data(self.on_error_replace_with())
     44                 return i + 1
     45             if self.on_error == self.ON_ERROR_SKIP:
     46                 return i + 1
     47             raise
     48 
     49     def unknown_decl(self, data):
     50         pass
     51 
     52     def goahead(self, end):
     53         rawdata = self.rawdata
     54         i = 0
     55         n = len(rawdata)
     56         while i < n:
     57             match = self.interesting.search(rawdata, i)
     58             if match:
     59                 j = match.start()
     60             else:
     61                 j = n
     62             if i < j:
     63                 self.handle_data(rawdata[i:j])
     64             i = self.updatepos(i, j)
     65             if i == n:
     66                 break
     67             startswith = rawdata.startswith
     68             if startswith('<', i):
     69                 if HTMLParser.starttagopen.match(rawdata, i):
     70                     k = self.parse_starttag(i)
     71                 elif startswith('</', i):
     72                     k = self.parse_endtag(i)
     73                 elif startswith('<!--', i):
     74                     k = self.parse_comment(i)
     75                 elif startswith('<?', i):
     76                     k = self.parse_pi(i)
     77                 elif startswith('<!', i):
     78                     k = self.parse_declaration(i)
     79                 elif i + 1 < n:
     80                     self.handle_data('<')
     81                     k = i + 1
     82                 else:
     83                     break
     84                 if k < 0:
     85                     if end:
     86                         self.error('EOF in middle of construct')
     87                     break
     88                 i = self.updatepos(i, k)
     89             elif startswith('&#', i):
     90                 match = HTMLParser.charref.match(rawdata, i)
     91                 if match:
     92                     name = match.group()[2:-1]
     93                     k = match.end()
     94                     if not startswith(';', k - 1):
     95                         self.handle_data('&#' + name)
     96                         k = k - 1
     97                     else:
     98                         self.handle_charref(name)
     99                     i = self.updatepos(i, k)
    100                     continue
    101                 else:
    102                     if ';' in rawdata[i:]:
    103                         self.handle_data(rawdata[0:2])
    104                         i = self.updatepos(i, 2)
    105                     break
    106             elif startswith('&', i):
    107                 match = HTMLParser.entityref.match(rawdata, i)
    108                 if match:
    109                     name = match.group(1)
    110                     k = match.end()
    111                     if not startswith(';', k - 1):
    112                         self.handle_data('&' + name)
    113                         k = k - 1
    114                     else:
    115                         self.handle_entityref(name)
    116                     i = self.updatepos(i, k)
    117                     continue
    118                 match = HTMLParser.incomplete.match(rawdata, i)
    119                 if match:
    120                     self.handle_data(rawdata[i:])
    121                     if end and match.group() == rawdata[i:]:
    122                         self.error('EOF in middle of entity or char ref')
    123                     break
    124                 elif i + 1 < n:
    125                     self.handle_data('&')
    126                     i = self.updatepos(i, i + 1)
    127                 else:
    128                     self.handle_data('&')
    129                     break
    130 
    131         if end and i < n:
    132             self.handle_data(rawdata[i:n])
    133             i = self.updatepos(i, n)
    134         self.rawdata = rawdata[i:]
    135 
    136 
    137 class TagStripper(TolerantParser):
    138 
    139     def __init__(self, tag_list = tuple(), preserve_content_list = tuple()):
    140         TolerantParser.__init__(self)
    141         self._stripped = []
    142         self.tag_list = tag_list
    143         self.preserve_content_list = preserve_content_list
    144         self.ignore_content_until = None
    145 
    146     def handle_data(self, data):
    147         if not self.ignore_content_until:
    148             self._stripped.append(data)
    149 
    150     def handle_starttag(self, tag, attrs):
    151         if tag in self.tag_list:
    152             if tag not in self.preserve_content_list:
    153                 self.ignore_content_until = tag
    154         else:
    155             attributes = ''
    156             if attrs:
    157                 attributes = ' %s' % attr_str(attrs)
    158             self._stripped.append('<%s%s>' % (tag, attributes))
    159 
    160     def handle_endtag(self, tag):
    161         if self.ignore_content_until == tag:
    162             self.ignore_content_until = None
    163         if tag not in self.tag_list:
    164             self._stripped.append('</%s>' % tag)
    165 
    166     def handle_entityref(self, name):
    167         if not self.ignore_content_until:
    168             self._stripped.append('&%s;' % name)
    169 
    170     def handle_charref(self, name):
    171         if not self.ignore_content_until:
    172             self._stripped.append('&#%s;' % name)
    173 
    174     def get_stripped(self):
    175         return ''.join(self._stripped)
    176 
    177     def flush_stripped(self):
    178         text = self.get_stripped()
    179         self._stripped = []
    180         return text
    181 
    182 
    183 class HTMLStripper(TolerantParser):
    184     HTML_WHITESPACE = '[ \\t\\r\\n]+'
    185     HTML_BREAKS = '( \\n|\\n )'
    186     HTML_NON_CONTENT_TAGS = ['head',
    187      'script',
    188      'style',
    189      'embed',
    190      'frameset',
    191      'object',
    192      'iframe',
    193      'source',
    194      'track',
    195      'video',
    196      'audio',
    197      'canvas',
    198      'meta']
    199 
    200     def __init__(self, preserve_links = True, preserve_images = False, decode_chars = True, error_tolerance = True):
    201         TolerantParser.__init__(self)
    202         self._text = []
    203         self.preserve_links = preserve_links
    204         self.preserve_images = preserve_images
    205         self.decode_chars = decode_chars
    206         self.error_tolerance = error_tolerance
    207         self.link = ''
    208         self.ignore_content_until = None
    209 
    210     def handle_data(self, data):
    211         if not self.ignore_content_until:
    212             self._text.append(data)
    213 
    214     def handle_starttag(self, tag, attrs):
    215         if tag in self.HTML_NON_CONTENT_TAGS:
    216             self.ignore_content_until = tag
    217         if tag == 'a':
    218             if self.preserve_links and attrs:
    219                 attrs = dict(attrs)
    220                 self.link = attrs.get('href', '')
    221         if tag == 'img':
    222             if self.preserve_images and attrs:
    223                 attrs = dict(attrs)
    224                 self._text.append('[%s]' % attrs.get('src', ''))
    225         elif tag == 'br':
    226             self._add_linefeed()
    227 
    228     def handle_endtag(self, tag):
    229         if self.ignore_content_until == tag:
    230             self.ignore_content_until = None
    231         if tag in ('p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
    232             self._add_linefeed(2)
    233         elif tag == 'a' and self.preserve_links and self.link:
    234             self._text.append(' [%s]' % self.link)
    235             self.link = ''
    236 
    237     def handle_entityref(self, name):
    238         if not self.ignore_content_until:
    239             code = '&%s;' % name
    240             if self.decode_chars:
    241                 code = self.unescape(code)
    242             self._text.append(code)
    243 
    244     def handle_charref(self, name):
    245         if not self.ignore_content_until:
    246             code = '&#%s;' % name
    247             if self.decode_chars:
    248                 code = self.unescape(code)
    249             self._text.append(code)
    250 
    251     def on_error_replace_with(self):
    252         if self.decode_chars:
    253             return '['
    254         else:
    255             return '&lt;'
    256 
    257     def _add_linefeed(self, count = 1):
    258         self._text.append('{{LF}}' * count)
    259 
    260     def get_text(self):
    261         text = ''.join(self._text)
    262         text = re.sub(self.HTML_WHITESPACE, ' ', text)
    263         text = text.replace('{{LF}}', '\n')
    264         text = re.sub(self.HTML_BREAKS, '\n', text)
    265         return text.strip()
    266 
    267     def flush_text(self):
    268         text = self.get_text()
    269         self._text = []
    270         return text
    271 
    272 
    273 class AbstractCleaner(object):
    274 
    275     def __init__(self, markup):
    276         self.markup = markup
    277         self.ok = True
    278         self.clean_markup = ''
    279         self.missing_tags = None
    280         self.break_idx = -1
    281         self.fixed_markup = None
    282         self.html = ''
    283         self.length = 0
    284 
    285     def cleanup(self):
    286         if self.markup and isinstance(self.markup, (str, unicode)):
    287             self.length = len(self.markup)
    288             self.ok, self.missing_tags, self.break_idx = validate_evemarkup(self.markup)
    289             self.clean_markup = remove_loc_tags(self.markup)
    290             self.clean_markup = remove_empty_formatting_tags(self.clean_markup)
    291             if not self.ok:
    292                 motd_ok2, missing_tags2, break_idx2 = validate_evemarkup(self.clean_markup)
    293                 if not motd_ok2:
    294                     self.fixed_markup = fix_broken_markup(self.clean_markup, missing_tags2, break_idx2)
    295                     self.html = evemarkup_to_html(self.fixed_markup)
    296                 else:
    297                     self.html = evemarkup_to_html(self.clean_markup)
    298             else:
    299                 self.html = evemarkup_to_html(self.clean_markup)
    300 
    301 
    302 class MotdCleaner(AbstractCleaner):
    303 
    304     def __init__(self, live_motd, channel):
    305         super(MotdCleaner, self).__init__(live_motd)
    306         self.channel = channel
    307         self.db_motd_length = 0
    308         self.has_loc = None
    309 
    310     def cleanup(self):
    311         if self.markup:
    312             self.has_loc = '<loc>' in self.markup
    313         else:
    314             self.has_loc = False
    315         if self.channel.motd:
    316             self.db_motd_length = len(self.channel.motd)
    317         super(MotdCleaner, self).cleanup()
    318 
    319 
    320 class EveMailCleaner(AbstractCleaner):
    321 
    322     def __init__(self, body):
    323         super(EveMailCleaner, self).__init__(body)
    324 
    325 
    326 def strip_html(html_string, preserve_links = True, preserve_images = False, decode_chars = True):
    327     stripper = HTMLStripper(preserve_links, preserve_images, decode_chars)
    328     stripper.feed(html_string)
    329     return stripper.get_text()
    330 
    331 
    332 def newline_to_html(string):
    333     return '<p>%s</p>' % reduce(lambda h, n: h.replace(*n), (('\r', ''), ('\n\n', '</p><p>'), ('\n', '<br />')), string)
    334 
    335 
    336 def sanitize(html_string, keep_amp = False):
    337     if not keep_amp:
    338         html_string = html_string.replace('&', '&amp;')
    339     return reduce(lambda string, params: string.replace(*params), (('<', '&lt;'), ('>', '&gt;')), html_string)
    340 
    341 
    342 def unsanitize(string):
    343     return reduce(lambda string, params: string.replace(*params), (('&gt;', '>'), ('&lt;', '<'), ('&amp;', '&')), string)
    344 
    345 
    346 def esc_email_tags(html, replacer = const.EMAIL_TAG_REPLACE_HTML):
    347     return const.EMAIL_TAG_GRABBER.sub(replacer, html)
    348 
    349 
    350 def strip_bad_tags(html_string):
    351     stripper = TagStripper(HTMLStripper.HTML_NON_CONTENT_TAGS)
    352     stripper.feed(html_string)
    353     return stripper.get_stripped()
    354 
    355 
    356 def attr_str(attribute_list, xhtml_strict = False):
    357     buff = []
    358     if isinstance(attribute_list, dict):
    359         attribute_list = attribute_list.items()
    360     for attr in attribute_list:
    361         if len(attr) > 1:
    362             buff.append('%s="%s"' % (attr[0], attr[1]))
    363         elif xhtml_strict:
    364             buff.append('%s="%s"' % (attr[0], attr[0]))
    365         else:
    366             buff.append('%s' % attr[0])
    367 
    368     return ' '.join(buff)
    369 
    370 
    371 def parse_user_notes(raw_notes, note_date = datetime.datetime.now()):
    372     raw_notes = sanitize(raw_notes, keep_amp=True)
    373     raw_notes = parse_markdown_link(raw_notes)
    374     note_date = datetimeutils.any_to_datetime(note_date)
    375     i = 0
    376     parsed_notes = []
    377     for m in const.NOTE_LINK_A_HREF_MATCHER.finditer(raw_notes):
    378         parsed_notes.append(parse_note_auto_link(raw_notes[i:m.start()], note_date))
    379         parsed_notes.append(m.group(0))
    380         i = m.end()
    381 
    382     parsed_notes.append(parse_note_auto_link(raw_notes[i:], note_date))
    383     return ''.join(parsed_notes).replace('\n', '<br>')
    384 
    385 
    386 def parse_markdown_link(raw_text):
    387     return re.sub(const.NOTE_LINK_MARKDOWN_PATTERN, const.NOTE_LINK_MARKDOWN_REPLACE, raw_text)
    388 
    389 
    390 def parse_note_auto_link(raw_text, note_date):
    391     i = 0
    392     parsed_text = []
    393     for m in const.NOTE_LINK_AUTO_LINKER.finditer(raw_text):
    394         parsed_text.append(raw_text[i:m.start()])
    395         parsed_text.append(const.NOTE_LINK_AUTO_LINK.format(url=_get_ticket_url(m.group(3), note_date), text='%s%s%s' % (m.group(1), m.group(2), m.group(3))))
    396         i = m.end()
    397 
    398     parsed_text.append(raw_text[i:])
    399     return ''.join(parsed_text)
    400 
    401 
    402 def _get_ticket_url(ticket_id, note_date):
    403     ticket_id = typeutils.int_eval(ticket_id)
    404     if ticket_id > const.OLD_TICKET_ID_THRESHOLD and note_date.year < 2016:
    405         return const.OLD_TICKET_URL.format(ticketID=ticket_id)
    406     return const.TICKET_URL.format(ticketID=ticket_id)
    407 
    408 
    409 def strip_extra_amp(string):
    410     return re.sub(const.EXTRA_AMP_MATCHER, const.EXTRA_AMP_REPLACER, string)
    411 
    412 
    413 def esp_item_link(item):
    414     if item.item_type.category_id == iconst.categoryOwner:
    415         if item.item_type.group_id == iconst.groupCharacter:
    416             return '/gm/character.py?action=Character&characterID=%d' % item.item_id
    417         if item.item_type.group_id == iconst.groupCorporation:
    418             return '/gm/corporation.py?action=Corporation&corporationID=%d' % item.item_id
    419         if item.item_type.group_id == iconst.groupAlliance:
    420             return '/gm/alliance.py?action=Alliance&allianceID=%d' % item.item_id
    421         if item.item_type.group_id == iconst.groupFaction:
    422             return '/gm/faction.py?action=Faction&factionID=%d' % item.item_id
    423     elif item.item_type.category_id == iconst.categoryCelestial:
    424         if item.item_type.group_id == iconst.groupRegion:
    425             return '/gd/universe.py?action=Region&regionID=%d' % item.item_id
    426         if item.item_type.group_id == iconst.groupConstellation:
    427             return '/gd/universe.py?action=Constellation&constellationID=%s' % item.item_id
    428         if item.item_type.group_id == iconst.groupSolarSystem:
    429             return '/gd/universe.py?action=System&systemID=%d' % item.item_id
    430         if item.item_type.group_id == iconst.groupAsteroidBelt:
    431             return '/gd/universe.py?action=AsteroidBelt&asteroidBeltID=%d' % item.item_id
    432         if item.item_type.group_id in [iconst.groupPlanet, iconst.groupMoon]:
    433             return '/gd/universe.py?action=Celestial&celestialID=%d' % item.item_id
    434     elif item.item_type.category_id == iconst.categoryStation:
    435         if item.item_type.group_id == iconst.groupStation:
    436             return '/gm/stations.py?action=Station&stationID=%d' % item.item_id
    437     else:
    438         if item.item_type.category_id == iconst.categorySovereigntyStructure:
    439             return '/gm/starbase.py?action=Starbase&towerID=%d' % item.item_id
    440         if item.item_type.category_id == iconst.categoryShip:
    441             return '/gm/character.py?action=Ship&characterID=%d&shipID=%d' % (item.owner_id, item.item_id)
    442     return '/gm/inventory.py?action=Item&itemID=%d' % item.item_id
    443 
    444 
    445 def validate_evemarkup(markup):
    446     markup = markup.lower()
    447     idx = -1
    448     if '<' in markup:
    449         idx = markup.rindex('<')
    450         last_part = markup[idx:]
    451         if '>' in last_part:
    452             idx = -1
    453     if idx > -1:
    454         markup = markup[:idx]
    455     start_less_markup = []
    456     tag_starts_found = []
    457     p = 0
    458     for match in re.finditer('<(\\w+)[^>]*>', markup):
    459         start_less_markup.append(markup[p:match.start()])
    460         p = match.end()
    461         tag = match.group(1)
    462         if tag != 'br':
    463             tag_starts_found.append(tag)
    464 
    465     start_less_markup.append(markup[p:])
    466     markup_no_start = ''.join(start_less_markup)
    467     for match in re.finditer('</(\\w+)>', markup_no_start):
    468         tag = match.group(1)
    469         if tag != 'br':
    470             if tag in tag_starts_found:
    471                 tag_starts_found.remove(tag)
    472 
    473     return (idx == -1 and not tag_starts_found, tag_starts_found, idx)
    474 
    475 
    476 def fix_broken_markup(markup, close_tags_list = None, index_of_broken_tag = -1):
    477     if index_of_broken_tag > -1:
    478         markup = markup[:index_of_broken_tag]
    479     if close_tags_list:
    480         append_tags = [ '</%s>' % t for t in close_tags_list ]
    481         markup = '%s%s' % (markup, ''.join(append_tags))
    482     return markup
    483 
    484 
    485 def remove_empty_formatting_tags(markup):
    486     tag_list = ('fontsize', 'color', 'i', 'u', 'b', 'url')
    487     for tag in tag_list:
    488         fixed_markup = []
    489         p = 0
    490         for match in re.finditer('(?i)<%s[^>]*></%s>' % (tag, tag), markup):
    491             fixed_markup.append(markup[p:match.start()])
    492             p = match.end()
    493 
    494         fixed_markup.append(markup[p:])
    495         markup = ''.join(fixed_markup)
    496 
    497     return markup
    498 
    499 
    500 def remove_tags(markup, *args):
    501     tag_list = []
    502     if args:
    503         for t in args:
    504             if isinstance(t, (list, tuple)):
    505                 tag_list.extend(t)
    506             else:
    507                 tag_list.append(t)
    508 
    509     for tag in tag_list:
    510         markup = re.sub('(?i)<%s[^>]*>(.*?)</%s>' % (tag, tag), '\\1', markup)
    511 
    512     return markup
    513 
    514 
    515 def remove_loc_tags(markup):
    516     return markup.replace('<loc>', '').replace('</loc>', '')
    517 
    518 
    519 def add_loc_tags(markup):
    520     return re.sub(const.SHOW_INFO_PATTERN, const.SHOW_INFO_REPLACE, markup)
    521 
    522 
    523 def evemarkup_to_html(markup):
    524     markup = remove_loc_tags(markup)
    525     markup = re.sub(const.EVEML_SHOWINFO_PATTERN, const.EVEML_SHOWINFO_REPLACE, markup)
    526     markup = re.sub(const.EVEML_FONT_PATTERN, const.EVEML_FONT_REPLACE, markup)
    527     markup = re.sub(const.EVEML_FONTSIZE_PATTERN, const.EVEML_FONTSIZE_REPLACE, markup)
    528     markup = re.sub(const.EVEML_COLOR_PATTERN, const.EVEML_COLOR_REPLACE, markup)
    529     markup = re.sub(const.EVEML_URL_PATTERN, const.EVEML_URL_REPLACE, markup)
    530     return markup
    531 
    532 
    533 def markup_syntax_html(markup):
    534     pass
    535 
    536 
    537 if __name__ == '__main__':
    538     s1 = "<fontsize=36>Stuff</fontsize><br><fontsize=36></fontsize><br><fontsize=14>* <b>Here's</b> a <loc><url=http://mbl.is>thin</loc>g</url><br>* Here's another <loc><url=showinfo:1373//90000012>thin</loc>g</url><br>* More <loc><url=showinfo:5//30000142>stuf</loc>f</url><br>* And <i>some</i> <loc><url=showinfo:54//60014839>stuff</url></loc> to <color=0xff00ff00>boot</color><br>* Wny &gt; no <loc><url=showinfo:2//98000002>type</url></loc>?!?<br>* Let's try <loc><color=0xffff0000>type</loc> again!</fontsize></color>"
    539     s2 = "<fontsize=36>Stuff</fontsize><br><br><fontsize=14>* <b>Here's</b> a <loc><url=http://mbl.is>thin</loc>g</url><br>* Here's another <loc><url=showinfo:1373//90000012>thin</loc>g</url><br>* More <loc><url=showinfo:5//30000142>stuf</loc>f</url><br>* And <i>some</i> <loc><url=showinfo:54//60014839>stuff</url></loc> to <color=0xff00ff00>boot</color><br>* Wny &gt; no <loc><url=showinfo:2//98000002>type</url></loc>?!?<br>* Let's try <loc><color=0xffff0000>type</loc> again!"
    540     s3 = "<fontsize=36>Stuff</fontsize><br><br><b>Here's</b> a <loc"
    541     s4 = "<fontsize=36>Stuff<br><br><b>Here's</b> a <loc"
    542     nl1 = remove_loc_tags(s1)
    543     nl2 = remove_loc_tags(s2)
    544     nl3 = remove_loc_tags(s3)
    545     nl4 = remove_loc_tags(s4)
    546     print('validate_evemarkup')
    547     ok1, tl1, ix1 = validate_evemarkup(nl1)
    548     print(ok1, tl1, ix1)
    549     ok2, tl2, ix2 = validate_evemarkup(nl2)
    550     print(ok2, tl2, ix2)
    551     ok3, tl3, ix3 = validate_evemarkup(nl3)
    552     print(ok3, tl3, ix3)
    553     ok4, tl4, ix4 = validate_evemarkup(nl4)
    554     print(ok4, tl4, ix4)
    555     print('fix_broken_markup')
    556     f1 = fix_broken_markup(nl1, tl1, ix1)
    557     print(f1)
    558     f2 = fix_broken_markup(nl3, tl3, ix3)
    559     print(f2)
    560     f3 = fix_broken_markup(nl2, tl2, ix2)
    561     print(f3)
    562     f4 = fix_broken_markup(nl4, tl4, ix4)
    563     print(f4)
    564     print('remove_empty_formatting_tags')
    565     ff1 = remove_empty_formatting_tags(f1)
    566     print(ff1)
    567     ff2 = remove_empty_formatting_tags(f2)
    568     print(ff2)
    569     ff3 = remove_empty_formatting_tags(f3)
    570     print(ff3)
    571     ff4 = remove_empty_formatting_tags(f4)
    572     print(ff4)
    573     print('add_loc_tags')
    574     fi1 = add_loc_tags(ff1)
    575     print(fi1)
    576     fi2 = add_loc_tags(ff2)
    577     print(fi2)
    578     fi3 = add_loc_tags(ff3)
    579     print(fi3)
    580     fi4 = add_loc_tags(ff4)
    581     print(fi4)
    582     print('----')
    583     print(remove_tags(fi1, 'fontsize', 'color'))