python模块:csv

  1 """
  2 csv.py - read/write/investigate CSV files
  3 """
  4 
  5 import re
  6 from _csv import Error, __version__, writer, reader, register_dialect, \
  7                  unregister_dialect, get_dialect, list_dialects, \
  8                  field_size_limit, \
  9                  QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \
 10                  __doc__
 11 from _csv import Dialect as _Dialect
 12 
 13 from collections import OrderedDict
 14 from io import StringIO
 15 
 16 __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE",
 17            "Error", "Dialect", "__doc__", "excel", "excel_tab",
 18            "field_size_limit", "reader", "writer",
 19            "register_dialect", "get_dialect", "list_dialects", "Sniffer",
 20            "unregister_dialect", "__version__", "DictReader", "DictWriter",
 21            "unix_dialect"]
 22 
 23 class Dialect:
 24     """Describe a CSV dialect.
 25 
 26     This must be subclassed (see csv.excel).  Valid attributes are:
 27     delimiter, quotechar, escapechar, doublequote, skipinitialspace,
 28     lineterminator, quoting.
 29 
 30     """
 31     _name = ""
 32     _valid = False
 33     # placeholders
 34     delimiter = None
 35     quotechar = None
 36     escapechar = None
 37     doublequote = None
 38     skipinitialspace = None
 39     lineterminator = None
 40     quoting = None
 41 
 42     def __init__(self):
 43         if self.__class__ != Dialect:
 44             self._valid = True
 45         self._validate()
 46 
 47     def _validate(self):
 48         try:
 49             _Dialect(self)
 50         except TypeError as e:
 51             # We do this for compatibility with py2.3
 52             raise Error(str(e))
 53 
 54 class excel(Dialect):
 55     """Describe the usual properties of Excel-generated CSV files."""
 56     delimiter = ','
 57     quotechar = '"'
 58     doublequote = True
 59     skipinitialspace = False
 60     lineterminator = '\r\n'
 61     quoting = QUOTE_MINIMAL
 62 register_dialect("excel", excel)
 63 
 64 class excel_tab(excel):
 65     """Describe the usual properties of Excel-generated TAB-delimited files."""
 66     delimiter = '\t'
 67 register_dialect("excel-tab", excel_tab)
 68 
 69 class unix_dialect(Dialect):
 70     """Describe the usual properties of Unix-generated CSV files."""
 71     delimiter = ','
 72     quotechar = '"'
 73     doublequote = True
 74     skipinitialspace = False
 75     lineterminator = '\n'
 76     quoting = QUOTE_ALL
 77 register_dialect("unix", unix_dialect)
 78 
 79 
 80 class DictReader:
 81     def __init__(self, f, fieldnames=None, restkey=None, restval=None,
 82                  dialect="excel", *args, **kwds):
 83         self._fieldnames = fieldnames   # list of keys for the dict
 84         self.restkey = restkey          # key to catch long rows
 85         self.restval = restval          # default value for short rows
 86         self.reader = reader(f, dialect, *args, **kwds)
 87         self.dialect = dialect
 88         self.line_num = 0
 89 
 90     def __iter__(self):
 91         return self
 92 
 93     @property
 94     def fieldnames(self):
 95         if self._fieldnames is None:
 96             try:
 97                 self._fieldnames = next(self.reader)
 98             except StopIteration:
 99                 pass
100         self.line_num = self.reader.line_num
101         return self._fieldnames
102 
103     @fieldnames.setter
104     def fieldnames(self, value):
105         self._fieldnames = value
106 
107     def __next__(self):
108         if self.line_num == 0:
109             # Used only for its side effect.
110             self.fieldnames
111         row = next(self.reader)
112         self.line_num = self.reader.line_num
113 
114         # unlike the basic reader, we prefer not to return blanks,
115         # because we will typically wind up with a dict full of None
116         # values
117         while row == []:
118             row = next(self.reader)
119         d = OrderedDict(zip(self.fieldnames, row))
120         lf = len(self.fieldnames)
121         lr = len(row)
122         if lf < lr:
123             d[self.restkey] = row[lf:]
124         elif lf > lr:
125             for key in self.fieldnames[lr:]:
126                 d[key] = self.restval
127         return d
128 
129 
130 class DictWriter:
131     def __init__(self, f, fieldnames, restval="", extrasaction="raise",
132                  dialect="excel", *args, **kwds):
133         self.fieldnames = fieldnames    # list of keys for the dict
134         self.restval = restval          # for writing short dicts
135         if extrasaction.lower() not in ("raise", "ignore"):
136             raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'"
137                              % extrasaction)
138         self.extrasaction = extrasaction
139         self.writer = writer(f, dialect, *args, **kwds)
140 
141     def writeheader(self):
142         header = dict(zip(self.fieldnames, self.fieldnames))
143         self.writerow(header)
144 
145     def _dict_to_list(self, rowdict):
146         if self.extrasaction == "raise":
147             wrong_fields = rowdict.keys() - self.fieldnames
148             if wrong_fields:
149                 raise ValueError("dict contains fields not in fieldnames: "
150                                  + ", ".join([repr(x) for x in wrong_fields]))
151         return (rowdict.get(key, self.restval) for key in self.fieldnames)
152 
153     def writerow(self, rowdict):
154         return self.writer.writerow(self._dict_to_list(rowdict))
155 
156     def writerows(self, rowdicts):
157         return self.writer.writerows(map(self._dict_to_list, rowdicts))
158 
159 # Guard Sniffer's type checking against builds that exclude complex()
160 try:
161     complex
162 except NameError:
163     complex = float
164 
165 class Sniffer:
166     '''
167     "Sniffs" the format of a CSV file (i.e. delimiter, quotechar)
168     Returns a Dialect object.
169     '''
170     def __init__(self):
171         # in case there is more than one possible delimiter
172         self.preferred = [',', '\t', ';', ' ', ':']
173 
174 
175     def sniff(self, sample, delimiters=None):
176         """
177         Returns a dialect (or None) corresponding to the sample
178         """
179 
180         quotechar, doublequote, delimiter, skipinitialspace = \
181                    self._guess_quote_and_delimiter(sample, delimiters)
182         if not delimiter:
183             delimiter, skipinitialspace = self._guess_delimiter(sample,
184                                                                 delimiters)
185 
186         if not delimiter:
187             raise Error("Could not determine delimiter")
188 
189         class dialect(Dialect):
190             _name = "sniffed"
191             lineterminator = '\r\n'
192             quoting = QUOTE_MINIMAL
193             # escapechar = ''
194 
195         dialect.doublequote = doublequote
196         dialect.delimiter = delimiter
197         # _csv.reader won't accept a quotechar of ''
198         dialect.quotechar = quotechar or '"'
199         dialect.skipinitialspace = skipinitialspace
200 
201         return dialect
202 
203 
204     def _guess_quote_and_delimiter(self, data, delimiters):
205         """
206         Looks for text enclosed between two identical quotes
207         (the probable quotechar) which are preceded and followed
208         by the same character (the probable delimiter).
209         For example:
210                          ,'some text',
211         The quote with the most wins, same with the delimiter.
212         If there is no quotechar the delimiter can't be determined
213         this way.
214         """
215 
216         matches = []
217         for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?",
218                       r'(?:^|\n)(?P["\']).*?(?P=quote)(?P[^\w\n"\'])(?P ?)',   #  ".*?",
219                       r'(?P>[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?:$|\n)',  # ,".*?"
220                       r'(?:^|\n)(?P["\']).*?(?P=quote)(?:$|\n)'):                            #  ".*?" (no delim, no space)
221             regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
222             matches = regexp.findall(data)
223             if matches:
224                 break
225 
226         if not matches:
227             # (quotechar, doublequote, delimiter, skipinitialspace)
228             return ('', False, None, 0)
229         quotes = {}
230         delims = {}
231         spaces = 0
232         groupindex = regexp.groupindex
233         for m in matches:
234             n = groupindex['quote'] - 1
235             key = m[n]
236             if key:
237                 quotes[key] = quotes.get(key, 0) + 1
238             try:
239                 n = groupindex['delim'] - 1
240                 key = m[n]
241             except KeyError:
242                 continue
243             if key and (delimiters is None or key in delimiters):
244                 delims[key] = delims.get(key, 0) + 1
245             try:
246                 n = groupindex['space'] - 1
247             except KeyError:
248                 continue
249             if m[n]:
250                 spaces += 1
251 
252         quotechar = max(quotes, key=quotes.get)
253 
254         if delims:
255             delim = max(delims, key=delims.get)
256             skipinitialspace = delims[delim] == spaces
257             if delim == '\n': # most likely a file with a single column
258                 delim = ''
259         else:
260             # there is *no* delimiter, it's a single column of quoted data
261             delim = ''
262             skipinitialspace = 0
263 
264         # if we see an extra quote between delimiters, we've got a
265         # double quoted format
266         dq_regexp = re.compile(
267                                r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \
268                                {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE)
269 
270 
271 
272         if dq_regexp.search(data):
273             doublequote = True
274         else:
275             doublequote = False
276 
277         return (quotechar, doublequote, delim, skipinitialspace)
278 
279 
280     def _guess_delimiter(self, data, delimiters):
281         """
282         The delimiter /should/ occur the same number of times on
283         each row. However, due to malformed data, it may not. We don't want
284         an all or nothing approach, so we allow for small variations in this
285         number.
286           1) build a table of the frequency of each character on every line.
287           2) build a table of frequencies of this frequency (meta-frequency?),
288              e.g.  'x occurred 5 times in 10 rows, 6 times in 1000 rows,
289              7 times in 2 rows'
290           3) use the mode of the meta-frequency to determine the /expected/
291              frequency for that character
292           4) find out how often the character actually meets that goal
293           5) the character that best meets its goal is the delimiter
294         For performance reasons, the data is evaluated in chunks, so it can
295         try and evaluate the smallest portion of the data possible, evaluating
296         additional chunks as necessary.
297         """
298 
299         data = list(filter(None, data.split('\n')))
300 
301         ascii = [chr(c) for c in range(127)] # 7-bit ASCII
302 
303         # build frequency tables
304         chunkLength = min(10, len(data))
305         iteration = 0
306         charFrequency = {}
307         modes = {}
308         delims = {}
309         start, end = 0, min(chunkLength, len(data))
310         while start < len(data):
311             iteration += 1
312             for line in data[start:end]:
313                 for char in ascii:
314                     metaFrequency = charFrequency.get(char, {})
315                     # must count even if frequency is 0
316                     freq = line.count(char)
317                     # value is the mode
318                     metaFrequency[freq] = metaFrequency.get(freq, 0) + 1
319                     charFrequency[char] = metaFrequency
320 
321             for char in charFrequency.keys():
322                 items = list(charFrequency[char].items())
323                 if len(items) == 1 and items[0][0] == 0:
324                     continue
325                 # get the mode of the frequencies
326                 if len(items) > 1:
327                     modes[char] = max(items, key=lambda x: x[1])
328                     # adjust the mode - subtract the sum of all
329                     # other frequencies
330                     items.remove(modes[char])
331                     modes[char] = (modes[char][0], modes[char][1]
332                                    - sum(item[1] for item in items))
333                 else:
334                     modes[char] = items[0]
335 
336             # build a list of possible delimiters
337             modeList = modes.items()
338             total = float(chunkLength * iteration)
339             # (rows of consistent data) / (number of rows) = 100%
340             consistency = 1.0
341             # minimum consistency threshold
342             threshold = 0.9
343             while len(delims) == 0 and consistency >= threshold:
344                 for k, v in modeList:
345                     if v[0] > 0 and v[1] > 0:
346                         if ((v[1]/total) >= consistency and
347                             (delimiters is None or k in delimiters)):
348                             delims[k] = v
349                 consistency -= 0.01
350 
351             if len(delims) == 1:
352                 delim = list(delims.keys())[0]
353                 skipinitialspace = (data[0].count(delim) ==
354                                     data[0].count("%c " % delim))
355                 return (delim, skipinitialspace)
356 
357             # analyze another chunkLength lines
358             start = end
359             end += chunkLength
360 
361         if not delims:
362             return ('', 0)
363 
364         # if there's more than one, fall back to a 'preferred' list
365         if len(delims) > 1:
366             for d in self.preferred:
367                 if d in delims.keys():
368                     skipinitialspace = (data[0].count(d) ==
369                                         data[0].count("%c " % d))
370                     return (d, skipinitialspace)
371 
372         # nothing else indicates a preference, pick the character that
373         # dominates(?)
374         items = [(v,k) for (k,v) in delims.items()]
375         items.sort()
376         delim = items[-1][1]
377 
378         skipinitialspace = (data[0].count(delim) ==
379                             data[0].count("%c " % delim))
380         return (delim, skipinitialspace)
381 
382 
383     def has_header(self, sample):
384         # Creates a dictionary of types of data in each column. If any
385         # column is of a single type (say, integers), *except* for the first
386         # row, then the first row is presumed to be labels. If the type
387         # can't be determined, it is assumed to be a string in which case
388         # the length of the string is the determining factor: if all of the
389         # rows except for the first are the same length, it's a header.
390         # Finally, a 'vote' is taken at the end for each column, adding or
391         # subtracting from the likelihood of the first row being a header.
392 
393         rdr = reader(StringIO(sample), self.sniff(sample))
394 
395         header = next(rdr) # assume first row is header
396 
397         columns = len(header)
398         columnTypes = {}
399         for i in range(columns): columnTypes[i] = None
400 
401         checked = 0
402         for row in rdr:
403             # arbitrary number of rows to check, to keep it sane
404             if checked > 20:
405                 break
406             checked += 1
407 
408             if len(row) != columns:
409                 continue # skip rows that have irregular number of columns
410 
411             for col in list(columnTypes.keys()):
412 
413                 for thisType in [int, float, complex]:
414                     try:
415                         thisType(row[col])
416                         break
417                     except (ValueError, OverflowError):
418                         pass
419                 else:
420                     # fallback to length of string
421                     thisType = len(row[col])
422 
423                 if thisType != columnTypes[col]:
424                     if columnTypes[col] is None: # add new column type
425                         columnTypes[col] = thisType
426                     else:
427                         # type is inconsistent, remove column from
428                         # consideration
429                         del columnTypes[col]
430 
431         # finally, compare results against first row and "vote"
432         # on whether it's a header
433         hasHeader = 0
434         for col, colType in columnTypes.items():
435             if type(colType) == type(0): # it's a length
436                 if len(header[col]) != colType:
437                     hasHeader += 1
438                 else:
439                     hasHeader -= 1
440             else: # attempt typecast
441                 try:
442                     colType(header[col])
443                 except (ValueError, TypeError):
444                     hasHeader += 1
445                 else:
446                     hasHeader -= 1
447 
448         return hasHeader > 0
csv

 

转载于:https://www.cnblogs.com/lmgsanm/p/8744331.html

你可能感兴趣的:(python模块:csv)