HTMLParser.py (9999B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\stdlib\HTMLParser.py 2 import markupbase 3 import re 4 interesting_normal = re.compile('[&<]') 5 interesting_cdata = re.compile('<(/|\\Z)') 6 incomplete = re.compile('&[a-zA-Z#]') 7 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') 8 charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') 9 starttagopen = re.compile('<[a-zA-Z]') 10 piclose = re.compile('>') 11 commentclose = re.compile('--\\s*>') 12 tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*') 13 attrfind = re.compile('\\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\\s*=\\s*(\\\'[^\\\']*\\\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$\\(\\)_#=~@]*))?') 14 locatestarttagend = re.compile('\n <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name\n (?:\\s+ # whitespace before attribute name\n (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name\n (?:\\s*=\\s* # value indicator\n (?:\'[^\']*\' # LITA-enclosed value\n |\\"[^\\"]*\\" # LIT-enclosed value\n |[^\'\\">\\s]+ # bare value\n )\n )?\n )\n )*\n \\s* # trailing whitespace\n', re.VERBOSE) 15 endendtag = re.compile('>') 16 endtagfind = re.compile('</\\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\\s*>') 17 18 class HTMLParseError(Exception): 19 20 def __init__(self, msg, position = (None, None)): 21 self.msg = msg 22 self.lineno = position[0] 23 self.offset = position[1] 24 25 def __str__(self): 26 result = self.msg 27 if self.lineno is not None: 28 result = result + ', at line %d' % self.lineno 29 if self.offset is not None: 30 result = result + ', column %d' % (self.offset + 1) 31 return result 32 33 34 class HTMLParser(markupbase.ParserBase): 35 CDATA_CONTENT_ELEMENTS = ('script', 'style') 36 37 def __init__(self): 38 self.reset() 39 40 def reset(self): 41 self.rawdata = '' 42 self.lasttag = '???' 43 self.interesting = interesting_normal 44 markupbase.ParserBase.reset(self) 45 46 def feed(self, data): 47 self.rawdata = self.rawdata + data 48 self.goahead(0) 49 50 def close(self): 51 self.goahead(1) 52 53 def error(self, message): 54 raise HTMLParseError(message, self.getpos()) 55 56 __starttag_text = None 57 58 def get_starttag_text(self): 59 return self.__starttag_text 60 61 def set_cdata_mode(self): 62 self.interesting = interesting_cdata 63 64 def clear_cdata_mode(self): 65 self.interesting = interesting_normal 66 67 def goahead(self, end): 68 rawdata = self.rawdata 69 i = 0 70 n = len(rawdata) 71 while i < n: 72 match = self.interesting.search(rawdata, i) 73 if match: 74 j = match.start() 75 else: 76 j = n 77 if i < j: 78 self.handle_data(rawdata[i:j]) 79 i = self.updatepos(i, j) 80 if i == n: 81 break 82 startswith = rawdata.startswith 83 if startswith('<', i): 84 if starttagopen.match(rawdata, i): 85 k = self.parse_starttag(i) 86 elif startswith('</', i): 87 k = self.parse_endtag(i) 88 elif startswith('<!--', i): 89 k = self.parse_comment(i) 90 elif startswith('<?', i): 91 k = self.parse_pi(i) 92 elif startswith('<!', i): 93 k = self.parse_declaration(i) 94 elif i + 1 < n: 95 self.handle_data('<') 96 k = i + 1 97 else: 98 break 99 if k < 0: 100 if end: 101 self.error('EOF in middle of construct') 102 break 103 i = self.updatepos(i, k) 104 elif startswith('&#', i): 105 match = charref.match(rawdata, i) 106 if match: 107 name = match.group()[2:-1] 108 self.handle_charref(name) 109 k = match.end() 110 if not startswith(';', k - 1): 111 k = k - 1 112 i = self.updatepos(i, k) 113 continue 114 else: 115 if ';' in rawdata[i:]: 116 self.handle_data(rawdata[0:2]) 117 i = self.updatepos(i, 2) 118 break 119 elif startswith('&', i): 120 match = entityref.match(rawdata, i) 121 if match: 122 name = match.group(1) 123 self.handle_entityref(name) 124 k = match.end() 125 if not startswith(';', k - 1): 126 k = k - 1 127 i = self.updatepos(i, k) 128 continue 129 match = incomplete.match(rawdata, i) 130 if match: 131 if end and match.group() == rawdata[i:]: 132 self.error('EOF in middle of entity or char ref') 133 break 134 elif i + 1 < n: 135 self.handle_data('&') 136 i = self.updatepos(i, i + 1) 137 else: 138 break 139 140 if end and i < n: 141 self.handle_data(rawdata[i:n]) 142 i = self.updatepos(i, n) 143 self.rawdata = rawdata[i:] 144 145 def parse_pi(self, i): 146 rawdata = self.rawdata 147 match = piclose.search(rawdata, i + 2) 148 if not match: 149 return -1 150 j = match.start() 151 self.handle_pi(rawdata[i + 2:j]) 152 j = match.end() 153 return j 154 155 def parse_starttag(self, i): 156 self.__starttag_text = None 157 endpos = self.check_for_whole_start_tag(i) 158 if endpos < 0: 159 return endpos 160 rawdata = self.rawdata 161 self.__starttag_text = rawdata[i:endpos] 162 attrs = [] 163 match = tagfind.match(rawdata, i + 1) 164 k = match.end() 165 self.lasttag = tag = rawdata[i + 1:k].lower() 166 while k < endpos: 167 m = attrfind.match(rawdata, k) 168 if not m: 169 break 170 attrname, rest, attrvalue = m.group(1, 2, 3) 171 if not rest: 172 attrvalue = None 173 elif attrvalue[:1] == "'" == attrvalue[-1:] or attrvalue[:1] == '"' == attrvalue[-1:]: 174 attrvalue = attrvalue[1:-1] 175 attrvalue = self.unescape(attrvalue) 176 attrs.append((attrname.lower(), attrvalue)) 177 k = m.end() 178 179 end = rawdata[k:endpos].strip() 180 if end not in ('>', '/>'): 181 lineno, offset = self.getpos() 182 if '\n' in self.__starttag_text: 183 lineno = lineno + self.__starttag_text.count('\n') 184 offset = len(self.__starttag_text) - self.__starttag_text.rfind('\n') 185 else: 186 offset = offset + len(self.__starttag_text) 187 self.error('junk characters in start tag: %r' % (rawdata[k:endpos][:20],)) 188 if end.endswith('/>'): 189 self.handle_startendtag(tag, attrs) 190 else: 191 self.handle_starttag(tag, attrs) 192 if tag in self.CDATA_CONTENT_ELEMENTS: 193 self.set_cdata_mode() 194 return endpos 195 196 def check_for_whole_start_tag(self, i): 197 rawdata = self.rawdata 198 m = locatestarttagend.match(rawdata, i) 199 if m: 200 j = m.end() 201 next = rawdata[j:j + 1] 202 if next == '>': 203 return j + 1 204 if next == '/': 205 if rawdata.startswith('/>', j): 206 return j + 2 207 if rawdata.startswith('/', j): 208 return -1 209 self.updatepos(i, j + 1) 210 self.error('malformed empty start tag') 211 if next == '': 212 return -1 213 if next in 'abcdefghijklmnopqrstuvwxyz=/ABCDEFGHIJKLMNOPQRSTUVWXYZ': 214 return -1 215 self.updatepos(i, j) 216 self.error('malformed start tag') 217 raise AssertionError('we should not get here!') 218 219 def parse_endtag(self, i): 220 rawdata = self.rawdata 221 match = endendtag.search(rawdata, i + 1) 222 if not match: 223 return -1 224 j = match.end() 225 match = endtagfind.match(rawdata, i) 226 if not match: 227 self.error('bad end tag: %r' % (rawdata[i:j],)) 228 tag = match.group(1) 229 self.handle_endtag(tag.lower()) 230 self.clear_cdata_mode() 231 return j 232 233 def handle_startendtag(self, tag, attrs): 234 self.handle_starttag(tag, attrs) 235 self.handle_endtag(tag) 236 237 def handle_starttag(self, tag, attrs): 238 pass 239 240 def handle_endtag(self, tag): 241 pass 242 243 def handle_charref(self, name): 244 pass 245 246 def handle_entityref(self, name): 247 pass 248 249 def handle_data(self, data): 250 pass 251 252 def handle_comment(self, data): 253 pass 254 255 def handle_decl(self, decl): 256 pass 257 258 def handle_pi(self, data): 259 pass 260 261 def unknown_decl(self, data): 262 self.error('unknown declaration: %r' % (data,)) 263 264 entitydefs = None 265 266 def unescape(self, s): 267 if '&' not in s: 268 return s 269 270 def replaceEntities(s): 271 s = s.groups()[0] 272 if s[0] == '#': 273 s = s[1:] 274 if s[0] in ('x', 'X'): 275 c = int(s[1:], 16) 276 else: 277 c = int(s) 278 return unichr(c) 279 import htmlentitydefs 280 if HTMLParser.entitydefs is None: 281 entitydefs = HTMLParser.entitydefs = {'apos': u"'"} 282 for k, v in htmlentitydefs.name2codepoint.iteritems(): 283 entitydefs[k] = unichr(v) 284 285 try: 286 return self.entitydefs[s] 287 except KeyError: 288 return '&' + s + ';' 289 290 return re.sub('&(#?[xX]?(?:[0-9a-fA-F]+|\\w{1,8}));', replaceEntities, s)