codecs.py (15146B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\stdlib\codecs.py 2 import __builtin__, sys 3 try: 4 from _codecs import * 5 except ImportError as why: 6 raise SystemError('Failed to load the builtin codecs: %s' % why) 7 8 __all__ = ['register', 9 'lookup', 10 'open', 11 'EncodedFile', 12 'BOM', 13 'BOM_BE', 14 'BOM_LE', 15 'BOM32_BE', 16 'BOM32_LE', 17 'BOM64_BE', 18 'BOM64_LE', 19 'BOM_UTF8', 20 'BOM_UTF16', 21 'BOM_UTF16_LE', 22 'BOM_UTF16_BE', 23 'BOM_UTF32', 24 'BOM_UTF32_LE', 25 'BOM_UTF32_BE', 26 'strict_errors', 27 'ignore_errors', 28 'replace_errors', 29 'xmlcharrefreplace_errors', 30 'register_error', 31 'lookup_error'] 32 BOM_UTF8 = '\xef\xbb\xbf' 33 BOM_LE = BOM_UTF16_LE = '\xff\xfe' 34 BOM_BE = BOM_UTF16_BE = '\xfe\xff' 35 BOM_UTF32_LE = '\xff\xfe\x00\x00' 36 BOM_UTF32_BE = '\x00\x00\xfe\xff' 37 if sys.byteorder == 'little': 38 BOM = BOM_UTF16 = BOM_UTF16_LE 39 BOM_UTF32 = BOM_UTF32_LE 40 else: 41 BOM = BOM_UTF16 = BOM_UTF16_BE 42 BOM_UTF32 = BOM_UTF32_BE 43 BOM32_LE = BOM_UTF16_LE 44 BOM32_BE = BOM_UTF16_BE 45 BOM64_LE = BOM_UTF32_LE 46 BOM64_BE = BOM_UTF32_BE 47 48 class CodecInfo(tuple): 49 50 def __new__(cls, encode, decode, streamreader = None, streamwriter = None, incrementalencoder = None, incrementaldecoder = None, name = None): 51 self = tuple.__new__(cls, (encode, 52 decode, 53 streamreader, 54 streamwriter)) 55 self.name = name 56 self.encode = encode 57 self.decode = decode 58 self.incrementalencoder = incrementalencoder 59 self.incrementaldecoder = incrementaldecoder 60 self.streamwriter = streamwriter 61 self.streamreader = streamreader 62 return self 63 64 def __repr__(self): 65 return '<%s.%s object for encoding %s at 0x%x>' % (self.__class__.__module__, 66 self.__class__.__name__, 67 self.name, 68 id(self)) 69 70 71 class Codec: 72 73 def encode(self, input, errors = 'strict'): 74 raise NotImplementedError 75 76 def decode(self, input, errors = 'strict'): 77 raise NotImplementedError 78 79 80 class IncrementalEncoder(object): 81 82 def __init__(self, errors = 'strict'): 83 self.errors = errors 84 self.buffer = '' 85 86 def encode(self, input, final = False): 87 raise NotImplementedError 88 89 def reset(self): 90 pass 91 92 def getstate(self): 93 return 0 94 95 def setstate(self, state): 96 pass 97 98 99 class BufferedIncrementalEncoder(IncrementalEncoder): 100 101 def __init__(self, errors = 'strict'): 102 IncrementalEncoder.__init__(self, errors) 103 self.buffer = '' 104 105 def _buffer_encode(self, input, errors, final): 106 raise NotImplementedError 107 108 def encode(self, input, final = False): 109 data = self.buffer + input 110 result, consumed = self._buffer_encode(data, self.errors, final) 111 self.buffer = data[consumed:] 112 return result 113 114 def reset(self): 115 IncrementalEncoder.reset(self) 116 self.buffer = '' 117 118 def getstate(self): 119 return self.buffer or 0 120 121 def setstate(self, state): 122 self.buffer = state or '' 123 124 125 class IncrementalDecoder(object): 126 127 def __init__(self, errors = 'strict'): 128 self.errors = errors 129 130 def decode(self, input, final = False): 131 raise NotImplementedError 132 133 def reset(self): 134 pass 135 136 def getstate(self): 137 return ('', 0) 138 139 def setstate(self, state): 140 pass 141 142 143 class BufferedIncrementalDecoder(IncrementalDecoder): 144 145 def __init__(self, errors = 'strict'): 146 IncrementalDecoder.__init__(self, errors) 147 self.buffer = '' 148 149 def _buffer_decode(self, input, errors, final): 150 raise NotImplementedError 151 152 def decode(self, input, final = False): 153 data = self.buffer + input 154 result, consumed = self._buffer_decode(data, self.errors, final) 155 self.buffer = data[consumed:] 156 return result 157 158 def reset(self): 159 IncrementalDecoder.reset(self) 160 self.buffer = '' 161 162 def getstate(self): 163 return (self.buffer, 0) 164 165 def setstate(self, state): 166 self.buffer = state[0] 167 168 169 class StreamWriter(Codec): 170 171 def __init__(self, stream, errors = 'strict'): 172 self.stream = stream 173 self.errors = errors 174 175 def write(self, object): 176 data, consumed = self.encode(object, self.errors) 177 self.stream.write(data) 178 179 def writelines(self, list): 180 self.write(''.join(list)) 181 182 def reset(self): 183 pass 184 185 def seek(self, offset, whence = 0): 186 self.stream.seek(offset, whence) 187 if whence == 0 and offset == 0: 188 self.reset() 189 190 def __getattr__(self, name, getattr = getattr): 191 return getattr(self.stream, name) 192 193 def __enter__(self): 194 return self 195 196 def __exit__(self, type, value, tb): 197 self.stream.close() 198 199 200 class StreamReader(Codec): 201 202 def __init__(self, stream, errors = 'strict'): 203 self.stream = stream 204 self.errors = errors 205 self.bytebuffer = '' 206 self.charbuffer = '' 207 self.linebuffer = None 208 209 def decode(self, input, errors = 'strict'): 210 raise NotImplementedError 211 212 def read(self, size = -1, chars = -1, firstline = False): 213 if self.linebuffer: 214 self.charbuffer = ''.join(self.linebuffer) 215 self.linebuffer = None 216 while True: 217 if chars < 0: 218 if size < 0: 219 if self.charbuffer: 220 break 221 elif len(self.charbuffer) >= size: 222 break 223 elif len(self.charbuffer) >= chars: 224 break 225 if size < 0: 226 newdata = self.stream.read() 227 else: 228 newdata = self.stream.read(size) 229 data = self.bytebuffer + newdata 230 try: 231 newchars, decodedbytes = self.decode(data, self.errors) 232 except UnicodeDecodeError as exc: 233 if firstline: 234 newchars, decodedbytes = self.decode(data[:exc.start], self.errors) 235 lines = newchars.splitlines(True) 236 if len(lines) <= 1: 237 raise 238 else: 239 raise 240 241 self.bytebuffer = data[decodedbytes:] 242 self.charbuffer += newchars 243 if not newdata: 244 break 245 246 if chars < 0: 247 result = self.charbuffer 248 self.charbuffer = '' 249 else: 250 result = self.charbuffer[:chars] 251 self.charbuffer = self.charbuffer[chars:] 252 return result 253 254 def readline(self, size = None, keepends = True): 255 if self.linebuffer: 256 line = self.linebuffer[0] 257 del self.linebuffer[0] 258 if len(self.linebuffer) == 1: 259 self.charbuffer = self.linebuffer[0] 260 self.linebuffer = None 261 if not keepends: 262 line = line.splitlines(False)[0] 263 return line 264 readsize = size or 72 265 line = '' 266 while True: 267 data = self.read(readsize, firstline=True) 268 if data: 269 if data.endswith('\r'): 270 data += self.read(size=1, chars=1) 271 line += data 272 lines = line.splitlines(True) 273 if lines: 274 if len(lines) > 1: 275 line = lines[0] 276 del lines[0] 277 if len(lines) > 1: 278 lines[-1] += self.charbuffer 279 self.linebuffer = lines 280 self.charbuffer = None 281 else: 282 self.charbuffer = lines[0] + self.charbuffer 283 if not keepends: 284 line = line.splitlines(False)[0] 285 break 286 line0withend = lines[0] 287 line0withoutend = lines[0].splitlines(False)[0] 288 if line0withend != line0withoutend: 289 self.charbuffer = ''.join(lines[1:]) + self.charbuffer 290 if keepends: 291 line = line0withend 292 else: 293 line = line0withoutend 294 break 295 if not data or size is not None: 296 if line and not keepends: 297 line = line.splitlines(False)[0] 298 break 299 if readsize < 8000: 300 readsize *= 2 301 302 return line 303 304 def readlines(self, sizehint = None, keepends = True): 305 data = self.read() 306 return data.splitlines(keepends) 307 308 def reset(self): 309 self.bytebuffer = '' 310 self.charbuffer = u'' 311 self.linebuffer = None 312 313 def seek(self, offset, whence = 0): 314 self.stream.seek(offset, whence) 315 self.reset() 316 317 def next(self): 318 line = self.readline() 319 if line: 320 return line 321 raise StopIteration 322 323 def __iter__(self): 324 return self 325 326 def __getattr__(self, name, getattr = getattr): 327 return getattr(self.stream, name) 328 329 def __enter__(self): 330 return self 331 332 def __exit__(self, type, value, tb): 333 self.stream.close() 334 335 336 class StreamReaderWriter: 337 encoding = 'unknown' 338 339 def __init__(self, stream, Reader, Writer, errors = 'strict'): 340 self.stream = stream 341 self.reader = Reader(stream, errors) 342 self.writer = Writer(stream, errors) 343 self.errors = errors 344 345 def read(self, size = -1): 346 return self.reader.read(size) 347 348 def readline(self, size = None): 349 return self.reader.readline(size) 350 351 def readlines(self, sizehint = None): 352 return self.reader.readlines(sizehint) 353 354 def next(self): 355 return self.reader.next() 356 357 def __iter__(self): 358 return self 359 360 def write(self, data): 361 return self.writer.write(data) 362 363 def writelines(self, list): 364 return self.writer.writelines(list) 365 366 def reset(self): 367 self.reader.reset() 368 self.writer.reset() 369 370 def seek(self, offset, whence = 0): 371 self.stream.seek(offset, whence) 372 self.reader.reset() 373 if whence == 0 and offset == 0: 374 self.writer.reset() 375 376 def __getattr__(self, name, getattr = getattr): 377 return getattr(self.stream, name) 378 379 def __enter__(self): 380 return self 381 382 def __exit__(self, type, value, tb): 383 self.stream.close() 384 385 386 class StreamRecoder: 387 data_encoding = 'unknown' 388 file_encoding = 'unknown' 389 390 def __init__(self, stream, encode, decode, Reader, Writer, errors = 'strict'): 391 self.stream = stream 392 self.encode = encode 393 self.decode = decode 394 self.reader = Reader(stream, errors) 395 self.writer = Writer(stream, errors) 396 self.errors = errors 397 398 def read(self, size = -1): 399 data = self.reader.read(size) 400 data, bytesencoded = self.encode(data, self.errors) 401 return data 402 403 def readline(self, size = None): 404 if size is None: 405 data = self.reader.readline() 406 else: 407 data = self.reader.readline(size) 408 data, bytesencoded = self.encode(data, self.errors) 409 return data 410 411 def readlines(self, sizehint = None): 412 data = self.reader.read() 413 data, bytesencoded = self.encode(data, self.errors) 414 return data.splitlines(1) 415 416 def next(self): 417 data = self.reader.next() 418 data, bytesencoded = self.encode(data, self.errors) 419 return data 420 421 def __iter__(self): 422 return self 423 424 def write(self, data): 425 data, bytesdecoded = self.decode(data, self.errors) 426 return self.writer.write(data) 427 428 def writelines(self, list): 429 data = ''.join(list) 430 data, bytesdecoded = self.decode(data, self.errors) 431 return self.writer.write(data) 432 433 def reset(self): 434 self.reader.reset() 435 self.writer.reset() 436 437 def __getattr__(self, name, getattr = getattr): 438 return getattr(self.stream, name) 439 440 def __enter__(self): 441 return self 442 443 def __exit__(self, type, value, tb): 444 self.stream.close() 445 446 447 def open(filename, mode = 'rb', encoding = None, errors = 'strict', buffering = 1): 448 if encoding is not None: 449 if 'U' in mode: 450 mode = mode.strip().replace('U', '') 451 if mode[:1] not in set('rwa'): 452 mode = 'r' + mode 453 if 'b' not in mode: 454 mode = mode + 'b' 455 file = __builtin__.open(filename, mode, buffering) 456 if encoding is None: 457 return file 458 info = lookup(encoding) 459 srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors) 460 srw.encoding = encoding 461 return srw 462 463 464 def EncodedFile(file, data_encoding, file_encoding = None, errors = 'strict'): 465 if file_encoding is None: 466 file_encoding = data_encoding 467 data_info = lookup(data_encoding) 468 file_info = lookup(file_encoding) 469 sr = StreamRecoder(file, data_info.encode, data_info.decode, file_info.streamreader, file_info.streamwriter, errors) 470 sr.data_encoding = data_encoding 471 sr.file_encoding = file_encoding 472 return sr 473 474 475 def getencoder(encoding): 476 return lookup(encoding).encode 477 478 479 def getdecoder(encoding): 480 return lookup(encoding).decode 481 482 483 def getincrementalencoder(encoding): 484 encoder = lookup(encoding).incrementalencoder 485 if encoder is None: 486 raise LookupError(encoding) 487 return encoder 488 489 490 def getincrementaldecoder(encoding): 491 decoder = lookup(encoding).incrementaldecoder 492 if decoder is None: 493 raise LookupError(encoding) 494 return decoder 495 496 497 def getreader(encoding): 498 return lookup(encoding).streamreader 499 500 501 def getwriter(encoding): 502 return lookup(encoding).streamwriter 503 504 505 def iterencode(iterator, encoding, errors = 'strict', **kwargs): 506 encoder = getincrementalencoder(encoding)(errors, **kwargs) 507 for input in iterator: 508 output = encoder.encode(input) 509 if output: 510 yield output 511 512 output = encoder.encode('', True) 513 if output: 514 yield output 515 516 517 def iterdecode(iterator, encoding, errors = 'strict', **kwargs): 518 decoder = getincrementaldecoder(encoding)(errors, **kwargs) 519 for input in iterator: 520 output = decoder.decode(input) 521 if output: 522 yield output 523 524 output = decoder.decode('', True) 525 if output: 526 yield output 527 528 529 def make_identity_dict(rng): 530 res = {} 531 for i in rng: 532 res[i] = i 533 534 return res 535 536 537 def make_encoding_map(decoding_map): 538 m = {} 539 for k, v in decoding_map.items(): 540 if v not in m: 541 m[v] = k 542 else: 543 m[v] = None 544 545 return m 546 547 548 try: 549 strict_errors = lookup_error('strict') 550 ignore_errors = lookup_error('ignore') 551 replace_errors = lookup_error('replace') 552 xmlcharrefreplace_errors = lookup_error('xmlcharrefreplace') 553 backslashreplace_errors = lookup_error('backslashreplace') 554 except LookupError: 555 strict_errors = None 556 ignore_errors = None 557 replace_errors = None 558 xmlcharrefreplace_errors = None 559 backslashreplace_errors = None 560 561 _false = 0 562 if _false: 563 import encodings 564 if __name__ == '__main__': 565 sys.stdout = EncodedFile(sys.stdout, 'latin-1', 'utf-8') 566 sys.stdin = EncodedFile(sys.stdin, 'utf-8', 'latin-1')