feedparser.py (11920B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\stdlib\email\feedparser.py 2 __all__ = ['FeedParser'] 3 import re 4 from email import errors 5 from email import message 6 NLCRE = re.compile('\r\n|\r|\n') 7 NLCRE_bol = re.compile('(\r\n|\r|\n)') 8 NLCRE_eol = re.compile('(\r\n|\r|\n)\\Z') 9 NLCRE_crack = re.compile('(\r\n|\r|\n)') 10 headerRE = re.compile('^(From |[\\041-\\071\\073-\\176]{1,}:|[\\t ])') 11 EMPTYSTRING = '' 12 NL = '\n' 13 NeedMoreData = object() 14 15 class BufferedSubFile(object): 16 17 def __init__(self): 18 self._partial = '' 19 self._lines = [] 20 self._eofstack = [] 21 self._closed = False 22 23 def push_eof_matcher(self, pred): 24 self._eofstack.append(pred) 25 26 def pop_eof_matcher(self): 27 return self._eofstack.pop() 28 29 def close(self): 30 self._lines.append(self._partial) 31 self._partial = '' 32 self._closed = True 33 34 def readline(self): 35 if not self._lines: 36 if self._closed: 37 return '' 38 return NeedMoreData 39 line = self._lines.pop() 40 for ateof in self._eofstack[::-1]: 41 if ateof(line): 42 self._lines.append(line) 43 return '' 44 45 return line 46 47 def unreadline(self, line): 48 self._lines.append(line) 49 50 def push(self, data): 51 data, self._partial = self._partial + data, '' 52 parts = NLCRE_crack.split(data) 53 self._partial = parts.pop() 54 if not self._partial and parts and parts[-1].endswith('\r'): 55 self._partial = parts.pop(-2) + parts.pop() 56 lines = [] 57 for i in range(len(parts) // 2): 58 lines.append(parts[i * 2] + parts[i * 2 + 1]) 59 60 self.pushlines(lines) 61 62 def pushlines(self, lines): 63 self._lines[:0] = lines[::-1] 64 65 def is_closed(self): 66 return self._closed 67 68 def __iter__(self): 69 return self 70 71 def next(self): 72 line = self.readline() 73 if line == '': 74 raise StopIteration 75 return line 76 77 78 class FeedParser: 79 80 def __init__(self, _factory = message.Message): 81 self._factory = _factory 82 self._input = BufferedSubFile() 83 self._msgstack = [] 84 self._parse = self._parsegen().next 85 self._cur = None 86 self._last = None 87 self._headersonly = False 88 89 def _set_headersonly(self): 90 self._headersonly = True 91 92 def feed(self, data): 93 self._input.push(data) 94 self._call_parse() 95 96 def _call_parse(self): 97 try: 98 self._parse() 99 except StopIteration: 100 pass 101 102 def close(self): 103 self._input.close() 104 self._call_parse() 105 root = self._pop_message() 106 if root.get_content_maintype() == 'multipart' and not root.is_multipart(): 107 root.defects.append(errors.MultipartInvariantViolationDefect()) 108 return root 109 110 def _new_message(self): 111 msg = self._factory() 112 if self._cur and self._cur.get_content_type() == 'multipart/digest': 113 msg.set_default_type('message/rfc822') 114 if self._msgstack: 115 self._msgstack[-1].attach(msg) 116 self._msgstack.append(msg) 117 self._cur = msg 118 self._last = msg 119 120 def _pop_message(self): 121 retval = self._msgstack.pop() 122 if self._msgstack: 123 self._cur = self._msgstack[-1] 124 else: 125 self._cur = None 126 return retval 127 128 def _parsegen(self): 129 self._new_message() 130 headers = [] 131 for line in self._input: 132 if line is NeedMoreData: 133 yield NeedMoreData 134 continue 135 if not headerRE.match(line): 136 if not NLCRE.match(line): 137 self._input.unreadline(line) 138 break 139 headers.append(line) 140 141 self._parse_headers(headers) 142 if self._headersonly: 143 lines = [] 144 while True: 145 line = self._input.readline() 146 if line is NeedMoreData: 147 yield NeedMoreData 148 continue 149 if line == '': 150 break 151 lines.append(line) 152 153 self._cur.set_payload(EMPTYSTRING.join(lines)) 154 return 155 if self._cur.get_content_type() == 'message/delivery-status': 156 while True: 157 self._input.push_eof_matcher(NLCRE.match) 158 for retval in self._parsegen(): 159 if retval is NeedMoreData: 160 yield NeedMoreData 161 continue 162 break 163 164 msg = self._pop_message() 165 self._input.pop_eof_matcher() 166 while True: 167 line = self._input.readline() 168 if line is NeedMoreData: 169 yield NeedMoreData 170 continue 171 break 172 173 while True: 174 line = self._input.readline() 175 if line is NeedMoreData: 176 yield NeedMoreData 177 continue 178 break 179 180 if line == '': 181 break 182 self._input.unreadline(line) 183 184 return 185 if self._cur.get_content_maintype() == 'message': 186 for retval in self._parsegen(): 187 if retval is NeedMoreData: 188 yield NeedMoreData 189 continue 190 break 191 192 self._pop_message() 193 return 194 if self._cur.get_content_maintype() == 'multipart': 195 boundary = self._cur.get_boundary() 196 if boundary is None: 197 self._cur.defects.append(errors.NoBoundaryInMultipartDefect()) 198 lines = [] 199 for line in self._input: 200 if line is NeedMoreData: 201 yield NeedMoreData 202 continue 203 lines.append(line) 204 205 self._cur.set_payload(EMPTYSTRING.join(lines)) 206 return 207 separator = '--' + boundary 208 boundaryre = re.compile('(?P<sep>' + re.escape(separator) + ')(?P<end>--)?(?P<ws>[ \\t]*)(?P<linesep>\\r\\n|\\r|\\n)?$') 209 capturing_preamble = True 210 preamble = [] 211 linesep = False 212 while True: 213 line = self._input.readline() 214 if line is NeedMoreData: 215 yield NeedMoreData 216 continue 217 if line == '': 218 break 219 mo = boundaryre.match(line) 220 if mo: 221 if mo.group('end'): 222 linesep = mo.group('linesep') 223 break 224 if capturing_preamble: 225 if preamble: 226 lastline = preamble[-1] 227 eolmo = NLCRE_eol.search(lastline) 228 if eolmo: 229 preamble[-1] = lastline[:-len(eolmo.group(0))] 230 self._cur.preamble = EMPTYSTRING.join(preamble) 231 capturing_preamble = False 232 self._input.unreadline(line) 233 continue 234 while True: 235 line = self._input.readline() 236 if line is NeedMoreData: 237 yield NeedMoreData 238 continue 239 mo = boundaryre.match(line) 240 if not mo: 241 self._input.unreadline(line) 242 break 243 244 self._input.push_eof_matcher(boundaryre.match) 245 for retval in self._parsegen(): 246 if retval is NeedMoreData: 247 yield NeedMoreData 248 continue 249 break 250 251 if self._last.get_content_maintype() == 'multipart': 252 epilogue = self._last.epilogue 253 if epilogue == '': 254 self._last.epilogue = None 255 elif epilogue is not None: 256 mo = NLCRE_eol.search(epilogue) 257 if mo: 258 end = len(mo.group(0)) 259 self._last.epilogue = epilogue[:-end] 260 else: 261 payload = self._last.get_payload() 262 if isinstance(payload, basestring): 263 mo = NLCRE_eol.search(payload) 264 if mo: 265 payload = payload[:-len(mo.group(0))] 266 self._last.set_payload(payload) 267 self._input.pop_eof_matcher() 268 self._pop_message() 269 self._last = self._cur 270 else: 271 preamble.append(line) 272 273 if capturing_preamble: 274 self._cur.defects.append(errors.StartBoundaryNotFoundDefect()) 275 self._cur.set_payload(EMPTYSTRING.join(preamble)) 276 epilogue = [] 277 for line in self._input: 278 if line is NeedMoreData: 279 yield NeedMoreData 280 continue 281 282 self._cur.epilogue = EMPTYSTRING.join(epilogue) 283 return 284 if linesep: 285 epilogue = [''] 286 else: 287 epilogue = [] 288 for line in self._input: 289 if line is NeedMoreData: 290 yield NeedMoreData 291 continue 292 epilogue.append(line) 293 294 if epilogue: 295 firstline = epilogue[0] 296 bolmo = NLCRE_bol.match(firstline) 297 if bolmo: 298 epilogue[0] = firstline[len(bolmo.group(0)):] 299 self._cur.epilogue = EMPTYSTRING.join(epilogue) 300 return 301 lines = [] 302 for line in self._input: 303 if line is NeedMoreData: 304 yield NeedMoreData 305 continue 306 lines.append(line) 307 308 self._cur.set_payload(EMPTYSTRING.join(lines)) 309 310 def _parse_headers(self, lines): 311 lastheader = '' 312 lastvalue = [] 313 for lineno, line in enumerate(lines): 314 if line[0] in ' \t': 315 if not lastheader: 316 defect = errors.FirstHeaderLineIsContinuationDefect(line) 317 self._cur.defects.append(defect) 318 continue 319 lastvalue.append(line) 320 continue 321 if lastheader: 322 lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n') 323 self._cur[lastheader] = lhdr 324 lastheader, lastvalue = '', [] 325 if line.startswith('From '): 326 if lineno == 0: 327 mo = NLCRE_eol.search(line) 328 if mo: 329 line = line[:-len(mo.group(0))] 330 self._cur.set_unixfrom(line) 331 continue 332 else: 333 if lineno == len(lines) - 1: 334 self._input.unreadline(line) 335 return 336 defect = errors.MisplacedEnvelopeHeaderDefect(line) 337 self._cur.defects.append(defect) 338 continue 339 i = line.find(':') 340 if i < 0: 341 defect = errors.MalformedHeaderDefect(line) 342 self._cur.defects.append(defect) 343 continue 344 lastheader = line[:i] 345 lastvalue = [line[i + 1:].lstrip()] 346 347 if lastheader: 348 self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n')