csv.py (11147B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\stdlib\csv.py 2 import re 3 from functools import reduce 4 from _csv import Error, __version__, writer, reader, register_dialect, unregister_dialect, get_dialect, list_dialects, field_size_limit, QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, __doc__ 5 from _csv import Dialect as _Dialect 6 try: 7 from cStringIO import StringIO 8 except ImportError: 9 from StringIO import StringIO 10 11 __all__ = ['QUOTE_MINIMAL', 12 'QUOTE_ALL', 13 'QUOTE_NONNUMERIC', 14 'QUOTE_NONE', 15 'Error', 16 'Dialect', 17 '__doc__', 18 'excel', 19 'excel_tab', 20 'field_size_limit', 21 'reader', 22 'writer', 23 'register_dialect', 24 'get_dialect', 25 'list_dialects', 26 'Sniffer', 27 'unregister_dialect', 28 '__version__', 29 'DictReader', 30 'DictWriter'] 31 32 class Dialect: 33 _name = '' 34 _valid = False 35 delimiter = None 36 quotechar = None 37 escapechar = None 38 doublequote = None 39 skipinitialspace = None 40 lineterminator = None 41 quoting = None 42 43 def __init__(self): 44 if self.__class__ != Dialect: 45 self._valid = True 46 self._validate() 47 48 def _validate(self): 49 try: 50 _Dialect(self) 51 except TypeError as e: 52 raise Error(str(e)) 53 54 55 class excel(Dialect): 56 delimiter = ',' 57 quotechar = '"' 58 doublequote = True 59 skipinitialspace = False 60 lineterminator = '\r\n' 61 quoting = QUOTE_MINIMAL 62 63 64 register_dialect('excel', excel) 65 66 class excel_tab(excel): 67 delimiter = '\t' 68 69 70 register_dialect('excel-tab', excel_tab) 71 72 class DictReader: 73 74 def __init__(self, f, fieldnames = None, restkey = None, restval = None, dialect = 'excel', *args, **kwds): 75 self._fieldnames = fieldnames 76 self.restkey = restkey 77 self.restval = restval 78 self.reader = reader(f, dialect, *args, **kwds) 79 self.dialect = dialect 80 self.line_num = 0 81 82 def __iter__(self): 83 return self 84 85 @property 86 def fieldnames(self): 87 if self._fieldnames is None: 88 try: 89 self._fieldnames = self.reader.next() 90 except StopIteration: 91 pass 92 93 self.line_num = self.reader.line_num 94 return self._fieldnames 95 96 @fieldnames.setter 97 def fieldnames(self, value): 98 self._fieldnames = value 99 100 def next(self): 101 if self.line_num == 0: 102 self.fieldnames 103 row = self.reader.next() 104 self.line_num = self.reader.line_num 105 while row == []: 106 row = self.reader.next() 107 108 d = dict(zip(self.fieldnames, row)) 109 lf = len(self.fieldnames) 110 lr = len(row) 111 if lf < lr: 112 d[self.restkey] = row[lf:] 113 elif lf > lr: 114 for key in self.fieldnames[lr:]: 115 d[key] = self.restval 116 117 return d 118 119 120 class DictWriter: 121 122 def __init__(self, f, fieldnames, restval = '', extrasaction = 'raise', dialect = 'excel', *args, **kwds): 123 self.fieldnames = fieldnames 124 self.restval = restval 125 if extrasaction.lower() not in ('raise', 'ignore'): 126 raise ValueError, "extrasaction (%s) must be 'raise' or 'ignore'" % extrasaction 127 self.extrasaction = extrasaction 128 self.writer = writer(f, dialect, *args, **kwds) 129 130 def writeheader(self): 131 header = dict(zip(self.fieldnames, self.fieldnames)) 132 self.writerow(header) 133 134 def _dict_to_list(self, rowdict): 135 if self.extrasaction == 'raise': 136 wrong_fields = [ k for k in rowdict if k not in self.fieldnames ] 137 if wrong_fields: 138 raise ValueError('dict contains fields not in fieldnames: ' + ', '.join(wrong_fields)) 139 return [ rowdict.get(key, self.restval) for key in self.fieldnames ] 140 141 def writerow(self, rowdict): 142 return self.writer.writerow(self._dict_to_list(rowdict)) 143 144 def writerows(self, rowdicts): 145 rows = [] 146 for rowdict in rowdicts: 147 rows.append(self._dict_to_list(rowdict)) 148 149 return self.writer.writerows(rows) 150 151 152 try: 153 complex 154 except NameError: 155 complex = float 156 157 class Sniffer: 158 159 def __init__(self): 160 self.preferred = [',', 161 '\t', 162 ';', 163 ' ', 164 ':'] 165 166 def sniff(self, sample, delimiters = None): 167 quotechar, doublequote, delimiter, skipinitialspace = self._guess_quote_and_delimiter(sample, delimiters) 168 if not delimiter: 169 delimiter, skipinitialspace = self._guess_delimiter(sample, delimiters) 170 if not delimiter: 171 raise Error, 'Could not determine delimiter' 172 173 class dialect(Dialect): 174 _name = 'sniffed' 175 lineterminator = '\r\n' 176 quoting = QUOTE_MINIMAL 177 178 dialect.doublequote = doublequote 179 dialect.delimiter = delimiter 180 dialect.quotechar = quotechar or '"' 181 dialect.skipinitialspace = skipinitialspace 182 return dialect 183 184 def _guess_quote_and_delimiter(self, data, delimiters): 185 matches = [] 186 for restr in ('(?P<delim>[^\\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\\w\n"\'])(?P<space> ?)', '(?P<delim>>[^\\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)', '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'): 187 regexp = re.compile(restr, re.DOTALL | re.MULTILINE) 188 matches = regexp.findall(data) 189 if matches: 190 break 191 192 if not matches: 193 return ('', 194 False, 195 None, 196 0) 197 quotes = {} 198 delims = {} 199 spaces = 0 200 for m in matches: 201 n = regexp.groupindex['quote'] - 1 202 key = m[n] 203 if key: 204 quotes[key] = quotes.get(key, 0) + 1 205 try: 206 n = regexp.groupindex['delim'] - 1 207 key = m[n] 208 except KeyError: 209 continue 210 211 if key and (delimiters is None or key in delimiters): 212 delims[key] = delims.get(key, 0) + 1 213 try: 214 n = regexp.groupindex['space'] - 1 215 except KeyError: 216 continue 217 218 if m[n]: 219 spaces += 1 220 221 quotechar = reduce(lambda a, b, quotes = quotes: quotes[a] > quotes[b] and a or b, quotes.keys()) 222 if delims: 223 delim = reduce(lambda a, b, delims = delims: delims[a] > delims[b] and a or b, delims.keys()) 224 skipinitialspace = delims[delim] == spaces 225 if delim == '\n': 226 delim = '' 227 else: 228 delim = '' 229 skipinitialspace = 0 230 dq_regexp = re.compile('((%(delim)s)|^)\\W*%(quote)s[^%(delim)s\\n]*%(quote)s[^%(delim)s\\n]*%(quote)s\\W*((%(delim)s)|$)' % {'delim': delim, 231 'quote': quotechar}, re.MULTILINE) 232 if dq_regexp.search(data): 233 doublequote = True 234 else: 235 doublequote = False 236 return (quotechar, 237 doublequote, 238 delim, 239 skipinitialspace) 240 241 def _guess_delimiter(self, data, delimiters): 242 data = filter(None, data.split('\n')) 243 ascii = [ chr(c) for c in range(127) ] 244 chunkLength = min(10, len(data)) 245 iteration = 0 246 charFrequency = {} 247 modes = {} 248 delims = {} 249 start, end = 0, min(chunkLength, len(data)) 250 while start < len(data): 251 iteration += 1 252 for line in data[start:end]: 253 for char in ascii: 254 metaFrequency = charFrequency.get(char, {}) 255 freq = line.count(char) 256 metaFrequency[freq] = metaFrequency.get(freq, 0) + 1 257 charFrequency[char] = metaFrequency 258 259 for char in charFrequency.keys(): 260 items = charFrequency[char].items() 261 if len(items) == 1 and items[0][0] == 0: 262 continue 263 if len(items) > 1: 264 modes[char] = reduce(lambda a, b: a[1] > b[1] and a or b, items) 265 items.remove(modes[char]) 266 modes[char] = (modes[char][0], modes[char][1] - reduce(lambda a, b: (0, a[1] + b[1]), items)[1]) 267 else: 268 modes[char] = items[0] 269 270 modeList = modes.items() 271 total = float(chunkLength * iteration) 272 consistency = 1.0 273 threshold = 0.9 274 while len(delims) == 0 and consistency >= threshold: 275 for k, v in modeList: 276 if v[0] > 0 and v[1] > 0: 277 if v[1] / total >= consistency and (delimiters is None or k in delimiters): 278 delims[k] = v 279 280 consistency -= 0.01 281 282 if len(delims) == 1: 283 delim = delims.keys()[0] 284 skipinitialspace = data[0].count(delim) == data[0].count('%c ' % delim) 285 return (delim, skipinitialspace) 286 start = end 287 end += chunkLength 288 289 if not delims: 290 return ('', 0) 291 if len(delims) > 1: 292 for d in self.preferred: 293 if d in delims.keys(): 294 skipinitialspace = data[0].count(d) == data[0].count('%c ' % d) 295 return (d, skipinitialspace) 296 297 items = [ (v, k) for k, v in delims.items() ] 298 items.sort() 299 delim = items[-1][1] 300 skipinitialspace = data[0].count(delim) == data[0].count('%c ' % delim) 301 return (delim, skipinitialspace) 302 303 def has_header(self, sample): 304 rdr = reader(StringIO(sample), self.sniff(sample)) 305 header = rdr.next() 306 columns = len(header) 307 columnTypes = {} 308 for i in range(columns): 309 columnTypes[i] = None 310 311 checked = 0 312 for row in rdr: 313 if checked > 20: 314 break 315 checked += 1 316 if len(row) != columns: 317 continue 318 for col in columnTypes.keys(): 319 for thisType in [int, 320 long, 321 float, 322 complex]: 323 try: 324 thisType(row[col]) 325 break 326 except (ValueError, OverflowError): 327 pass 328 329 else: 330 thisType = len(row[col]) 331 332 if thisType == long: 333 thisType = int 334 if thisType != columnTypes[col]: 335 if columnTypes[col] is None: 336 columnTypes[col] = thisType 337 else: 338 del columnTypes[col] 339 340 hasHeader = 0 341 for col, colType in columnTypes.items(): 342 if type(colType) == type(0): 343 if len(header[col]) != colType: 344 hasHeader += 1 345 else: 346 hasHeader -= 1 347 else: 348 try: 349 colType(header[col]) 350 except (ValueError, TypeError): 351 hasHeader += 1 352 else: 353 hasHeader -= 1 354 355 return hasHeader > 0