codekingpro/portable-devtools

1"""2PYTHON SOFTWARE FOUNDATION LICENSE VERSION 23--------------------------------------------4 51. This LICENSE AGREEMENT is between the Python Software Foundation6("PSF"), and the Individual or Organization ("Licensee") accessing and7otherwise using this software ("Python") in source or binary form and8its associated documentation.9 102. Subject to the terms and conditions of this License Agreement, PSF hereby11grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,12analyze, test, perform and/or display publicly, prepare derivative works,13distribute, and otherwise use Python alone or in any derivative version,14provided, however, that PSF's License Agreement and PSF's notice of copyright,15i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,2010162011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights17Reserved" are retained in Python alone or in any derivative version prepared by18Licensee.19 203. In the event Licensee prepares a derivative work that is based on21or incorporates Python or any part thereof, and wants to make22the derivative work available to others as provided herein, then23Licensee hereby agrees to include in any such work a brief summary of24the changes made to Python.25 264. PSF is making Python available to Licensee on an "AS IS"27basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR28IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND29DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS30FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT31INFRINGE ANY THIRD PARTY RIGHTS.32 335. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON34FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS35A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,36OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.37 386. This License Agreement will automatically terminate upon a material39breach of its terms and conditions.40 417. Nothing in this License Agreement shall be deemed to create any42relationship of agency, partnership, or joint venture between PSF and43Licensee.  This License Agreement does not grant permission to use PSF44trademarks or trade name in a trademark sense to endorse or promote45products or services of Licensee, or any third party.46 478. By copying, installing or otherwise using Python, Licensee48agrees to be bound by the terms and conditions of this License49Agreement.50"""51 52############################################################################53# Changes:54# Added new parameter in dialect 'replace_nulls_with' to compare it against55# the value to be quoted or not.56# Handle the null value if value is None or equal to57# 'replace_nulls_with' then it represents the null value, so no need to58# quote it.59############################################################################60 61__all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE",62           "Error", "Dialect", "__doc__", "Excel", "ExcelTab",63           "field_size_limit", "Reader", "Writer", "register_dialect",64           "get_dialect", "list_dialects", "unregister_dialect",65           "__version__", "DictReader", "DictWriter"]66 67import re68import numbers69from io import StringIO70from csv import (71    QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE,72    __version__, __doc__, Error, field_size_limit,73)74 75 76class QuoteStrategy():77    quoting = None78 79    def __init__(self, dialect):80        if self.quoting is not None:81            assert dialect.quoting == self.quoting82        self.dialect = dialect83        self.setup()84 85        escape_pattern_quoted = r'({quotechar})'.format(86            quotechar=re.escape(self.dialect.quotechar or '"'))87        escape_pattern_unquoted = r'([{specialchars}])'.format(88            specialchars=re.escape(self.specialchars))89 90        self.escape_re_quoted = re.compile(escape_pattern_quoted)91        self.escape_re_unquoted = re.compile(escape_pattern_unquoted)92 93    def setup(self):94        """Optional method for strategy-wide optimizations."""95 96    def quoted(self, field=None, raw_field=None, only=None):97        """Determine whether this field should be quoted."""98        raise NotImplementedError(99            'quoted must be implemented by a subclass')100 101    @property102    def specialchars(self):103        """The special characters that need to be escaped."""104        raise NotImplementedError(105            'specialchars must be implemented by a subclass')106 107    def escape_re(self, quoted=None):108        if quoted:109            return self.escape_re_quoted110        return self.escape_re_unquoted111 112    def escapechar(self, quoted=None):113        if quoted and self.dialect.doublequote:114            return self.dialect.quotechar115        return self.dialect.escapechar116 117    def prepare(self, raw_field, only=None):118        field = str(raw_field if raw_field is not None else '')119        quoted = self.quoted(field=field, raw_field=raw_field, only=only)120 121        escape_re = self.escape_re(quoted=quoted)122        escapechar = self.escapechar(quoted=quoted)123 124        if escape_re.search(field):125            escapechar = '\\\\' if escapechar == '\\' else escapechar126            if escapechar:127                escape_replace = \128                    r'{escapechar}\1'.format(escapechar=escapechar)129                field = escape_re.sub(escape_replace, field)130 131        if quoted:132            field = '{quotechar}{field}{quotechar}'.format(133                quotechar=self.dialect.quotechar, field=field)134 135        return field136 137 138class QuoteMinimalStrategy(QuoteStrategy):139    quoting = QUOTE_MINIMAL140 141    def setup(self):142        self.quoted_re = re.compile(r'[{specialchars}]'.format(143            specialchars=re.escape(self.specialchars)))144 145    @property146    def specialchars(self):147        return (148            self.dialect.lineterminator +149            self.dialect.quotechar +150            self.dialect.delimiter +151            (self.dialect.escapechar or '')152        )153 154    def quoted(self, field, only, **kwargs):155        if field == self.dialect.quotechar and not self.dialect.doublequote:156            # If the only character in the field is the quotechar, and157            # doublequote is false, then just escape without outer quotes.158            return False159        return field == '' and only or bool(self.quoted_re.search(field))160 161 162class QuoteAllStrategy(QuoteStrategy):163    quoting = QUOTE_ALL164 165    @property166    def specialchars(self):167        return self.dialect.quotechar168 169    def quoted(self, raw_field, **kwargs):170        # Handle the null value if raw_field is None or equal to171        # replace_nulls_with then it represents the null value, so no need to172        # quote it.173        if raw_field is None or raw_field == self.dialect.replace_nulls_with:174            return False175        return True176 177 178class QuoteNonnumericStrategy(QuoteStrategy):179    quoting = QUOTE_NONNUMERIC180 181    @property182    def specialchars(self):183        return (184            self.dialect.lineterminator +185            self.dialect.quotechar +186            self.dialect.delimiter +187            (self.dialect.escapechar or '')188        )189 190    def quoted(self, raw_field, **kwargs):191        # Handle the null value if raw_field is None or equal to192        # replace_nulls_with then it represents the null value, so no need to193        # quote it.194        if raw_field is None or raw_field == self.dialect.replace_nulls_with:195            return False196        return not isinstance(raw_field, numbers.Number)197 198 199class QuoteNoneStrategy(QuoteStrategy):200    quoting = QUOTE_NONE201 202    @property203    def specialchars(self):204        return (205            self.dialect.lineterminator +206            (self.dialect.quotechar or '') +207            self.dialect.delimiter +208            (self.dialect.escapechar or '')209        )210 211    def quoted(self, field, only, **kwargs):212        if field == '' and only:213            raise Error('single empty field record must be quoted')214        return False215 216 217class Writer():218    def __init__(self, fileobj, dialect='excel', **fmtparams):219        if fileobj is None:220            raise TypeError('fileobj must be file-like, not None')221 222        self.fileobj = fileobj223 224        if isinstance(dialect, str):225            dialect = get_dialect(dialect)226 227        try:228            self.dialect = Dialect.combine(dialect, fmtparams)229        except Error as e:230            raise TypeError(*e.args)231 232        strategies = {233            QUOTE_MINIMAL: QuoteMinimalStrategy,234            QUOTE_ALL: QuoteAllStrategy,235            QUOTE_NONNUMERIC: QuoteNonnumericStrategy,236            QUOTE_NONE: QuoteNoneStrategy,237        }238        self.strategy = strategies[self.dialect.quoting](self.dialect)239 240    def writerow(self, row):241        if row is None:242            raise Error('row must be an iterable')243 244        row = list(row)245        only = len(row) == 1246        row = [self.strategy.prepare(field, only=only) for field in row]247 248        line = self.dialect.delimiter.join(row) + self.dialect.lineterminator249        return self.fileobj.write(line)250 251    def writerows(self, rows):252        for row in rows:253            self.writerow(row)254 255 256START_RECORD = 0257START_FIELD = 1258ESCAPED_CHAR = 2259IN_FIELD = 3260IN_QUOTED_FIELD = 4261ESCAPE_IN_QUOTED_FIELD = 5262QUOTE_IN_QUOTED_FIELD = 6263EAT_CRNL = 7264AFTER_ESCAPED_CRNL = 8265 266 267class Reader():268    def __init__(self, fileobj, dialect='excel', **fmtparams):269        self.input_iter = iter(fileobj)270 271        if isinstance(dialect, str):272            dialect = get_dialect(dialect)273 274        try:275            self.dialect = Dialect.combine(dialect, fmtparams)276        except Error as e:277            raise TypeError(*e.args)278 279        self.fields = None280        self.field = None281        self.line_num = 0282 283    def parse_reset(self):284        self.fields = []285        self.field = []286        self.state = START_RECORD287        self.numeric_field = False288 289    def parse_save_field(self):290        field = ''.join(self.field)291        self.field = []292        if self.numeric_field:293            field = float(field)294            self.numeric_field = False295        self.fields.append(field)296 297    def parse_add_char(self, c):298        if len(self.field) >= field_size_limit():299            raise Error('field size limit exceeded')300        self.field.append(c)301 302    def parse_process_char(self, c):303        switch = {304            START_RECORD: self._parse_start_record,305            START_FIELD: self._parse_start_field,306            ESCAPED_CHAR: self._parse_escaped_char,307            AFTER_ESCAPED_CRNL: self._parse_after_escaped_crnl,308            IN_FIELD: self._parse_in_field,309            IN_QUOTED_FIELD: self._parse_in_quoted_field,310            ESCAPE_IN_QUOTED_FIELD: self._parse_escape_in_quoted_field,311            QUOTE_IN_QUOTED_FIELD: self._parse_quote_in_quoted_field,312            EAT_CRNL: self._parse_eat_crnl,313        }314        return switch[self.state](c)315 316    def _parse_start_record(self, c):317        if c == '\0':318            return319        elif c == '\n' or c == '\r':320            self.state = EAT_CRNL321            return322 323        self.state = START_FIELD324        return self._parse_start_field(c)325 326    def _parse_start_field(self, c):327        if c == '\n' or c == '\r' or c == '\0':328            self.parse_save_field()329            self.state = START_RECORD if c == '\0' else EAT_CRNL330        elif (c == self.dialect.quotechar and331              self.dialect.quoting != QUOTE_NONE):332            self.state = IN_QUOTED_FIELD333        elif c == self.dialect.escapechar:334            self.state = ESCAPED_CHAR335        elif c == ' ' and self.dialect.skipinitialspace:336            pass  # Ignore space at start of field337        elif c == self.dialect.delimiter:338            # Save empty field339            self.parse_save_field()340        else:341            # Begin new unquoted field342            if self.dialect.quoting == QUOTE_NONNUMERIC:343                self.numeric_field = True344            self.parse_add_char(c)345            self.state = IN_FIELD346 347    def _parse_escaped_char(self, c):348        if c == '\n' or c == '\r':349            self.parse_add_char(c)350            self.state = AFTER_ESCAPED_CRNL351            return352        if c == '\0':353            c = '\n'354        self.parse_add_char(c)355        self.state = IN_FIELD356 357    def _parse_after_escaped_crnl(self, c):358        if c == '\0':359            return360        return self._parse_in_field(c)361 362    def _parse_in_field(self, c):363        # In unquoted field364        if c == '\n' or c == '\r' or c == '\0':365            # End of line - return [fields]366            self.parse_save_field()367            self.state = START_RECORD if c == '\0' else EAT_CRNL368        elif c == self.dialect.escapechar:369            self.state = ESCAPED_CHAR370        elif c == self.dialect.delimiter:371            self.parse_save_field()372            self.state = START_FIELD373        else:374            # Normal character - save in field375            self.parse_add_char(c)376 377    def _parse_in_quoted_field(self, c):378        if c != '\0' and c == self.dialect.escapechar:379            self.state = ESCAPE_IN_QUOTED_FIELD380        elif c != '\0' and (c == self.dialect.quotechar and381                            self.dialect.quoting != QUOTE_NONE):382            if self.dialect.doublequote:383                self.state = QUOTE_IN_QUOTED_FIELD384            else:385                self.state = IN_FIELD386        elif c != '\0':387            self.parse_add_char(c)388 389    def _parse_escape_in_quoted_field(self, c):390        if c == '\0':391            c = '\n'392 393        self.parse_add_char(c)394        self.state = IN_QUOTED_FIELD395 396    def _parse_quote_in_quoted_field(self, c):397        if (self.dialect.quoting != QUOTE_NONE and398                c == self.dialect.quotechar):399            # save "" as "400            self.parse_add_char(c)401            self.state = IN_QUOTED_FIELD402        elif c == self.dialect.delimiter:403            self.parse_save_field()404            self.state = START_FIELD405        elif c == '\n' or c == '\r' or c == '\0':406            # End of line = return [fields]407            self.parse_save_field()408            self.state = START_RECORD if c == '\0' else EAT_CRNL409        elif not self.dialect.strict:410            self.parse_add_char(c)411            self.state = IN_FIELD412        else:413            # illegal414            raise Error("{delimiter}' expected after '{quotechar}".format(415                delimiter=self.dialect.delimiter,416                quotechar=self.dialect.quotechar,417            ))418 419    def _parse_eat_crnl(self, c):420        if c != '\n' and c != '\r' and c == '\0':421            self.state = START_RECORD422        elif c != '\n' and c != '\r':423            raise Error('new-line character seen in unquoted field - do you '424                        'need to open the file in universal-newline mode?')425 426    def __iter__(self):427        return self428 429    def __next__(self):430        self.parse_reset()431 432        while True:433            try:434                lineobj = next(self.input_iter)435            except StopIteration:436                if len(self.field) != 0 or self.state == IN_QUOTED_FIELD:437                    if self.dialect.strict:438                        raise Error('unexpected end of data')439                    self.parse_save_field()440                if self.fields:441                    break442                raise443 444            if not isinstance(lineobj, str):445                typ = type(lineobj)446                typ_name = 'bytes' if typ == bytes else typ.__name__447                err_str = ('iterator should return strings, not {0}'448                           ' (did you open the file in text mode?)')449                raise Error(err_str.format(typ_name))450 451            self.line_num += 1452            for c in lineobj:453                if c == '\0':454                    raise Error('line contains NULL byte')455                self.parse_process_char(c)456 457            self.parse_process_char('\0')458 459            if self.state == START_RECORD:460                break461 462        fields = self.fields463        self.fields = None464        return fields465 466    next = __next__467 468 469_dialect_registry = {}470 471 472def register_dialect(name, dialect='excel', **fmtparams):473    if not isinstance(name, str):474        raise TypeError('"name" must be a string')475 476    dialect = Dialect.extend(dialect, fmtparams)477 478    try:479        Dialect.validate(dialect)480    except Exception:481        raise TypeError('dialect is invalid')482 483    assert name not in _dialect_registry484    _dialect_registry[name] = dialect485 486 487def unregister_dialect(name):488    try:489        _dialect_registry.pop(name)490    except KeyError:491        raise Error('"{name}" not a registered dialect'.format(name=name))492 493 494def get_dialect(name):495    try:496        return _dialect_registry[name]497    except KeyError:498        raise Error('Could not find dialect {0}'.format(name))499 500 501def list_dialects():502    return list(_dialect_registry)503 504 505class Dialect():506    """Describe a CSV dialect.507    This must be subclassed (see csv.excel).  Valid attributes are:508    delimiter, quotechar, escapechar, doublequote, skipinitialspace,509    lineterminator, quoting, strict.510    """511    _name = ""512    _valid = False513    # placeholders514    delimiter = None515    quotechar = None516    escapechar = None517    doublequote = None518    skipinitialspace = None519    lineterminator = None520    quoting = None521    strict = None522 523    def __init__(self):524        self.validate(self)525        if self.__class__ != Dialect:526            self._valid = True527 528    @classmethod529    def validate(cls, dialect):530        dialect = cls.extend(dialect)531 532        if not isinstance(dialect.quoting, int):533            raise Error('"quoting" must be an integer')534 535        if dialect.delimiter is None:536            raise Error('delimiter must be set')537        cls.validate_text(dialect, 'delimiter')538 539        if dialect.lineterminator is None:540            raise Error('lineterminator must be set')541        if not isinstance(dialect.lineterminator, str):542            raise Error('"lineterminator" must be a string')543 544        if dialect.quoting not in [545                QUOTE_NONE, QUOTE_MINIMAL, QUOTE_NONNUMERIC, QUOTE_ALL]:546            raise Error('Invalid quoting specified')547 548        if dialect.quoting != QUOTE_NONE:549            if dialect.quotechar is None and dialect.escapechar is None:550                raise Error('quotechar must be set if quoting enabled')551            if dialect.quotechar is not None:552                cls.validate_text(dialect, 'quotechar')553 554    @staticmethod555    def validate_text(dialect, attr):556        val = getattr(dialect, attr)557        if not isinstance(val, str):558            if isinstance(val, bytes):559                raise Error('"{0}" must be string, not bytes'.format(attr))560            raise Error('"{0}" must be string, not {1}'.format(561                attr, type(val).__name__))562 563        if len(val) != 1:564            raise Error('"{0}" must be a 1-character string'.format(attr))565 566    @staticmethod567    def defaults():568        return {569            'delimiter': ',',570            'doublequote': True,571            'escapechar': None,572            'lineterminator': '\r\n',573            'quotechar': '"',574            'quoting': QUOTE_MINIMAL,575            'skipinitialspace': False,576            'strict': False,577            'replace_nulls_with': None578        }579 580    @classmethod581    def extend(cls, dialect, fmtparams=None):582        if isinstance(dialect, str):583            dialect = get_dialect(dialect)584 585        if fmtparams is None:586            return dialect587 588        defaults = cls.defaults()589 590        if any(param not in defaults for param in fmtparams):591            raise TypeError('Invalid fmtparam')592 593        specified = dict(594            (attr, getattr(dialect, attr, None))595            for attr in cls.defaults()596        )597 598        specified.update(fmtparams)599        return type(str('ExtendedDialect'), (cls,), specified)600 601    @classmethod602    def combine(cls, dialect, fmtparams):603        """Create a new dialect with defaults and added parameters."""604        dialect = cls.extend(dialect, fmtparams)605        defaults = cls.defaults()606        specified = dict(607            (attr, getattr(dialect, attr, None))608            for attr in defaults609            if getattr(dialect, attr, None) is not None or610            attr in ['quotechar', 'delimiter', 'lineterminator', 'quoting']611        )612 613        defaults.update(specified)614        dialect = type(str('CombinedDialect'), (cls,), defaults)615        cls.validate(dialect)616        return dialect()617 618    def __delattr__(self, attr):619        if self._valid:620            raise AttributeError('dialect is immutable.')621        super().__delattr__(attr)622 623    def __setattr__(self, attr, value):624        if self._valid:625            raise AttributeError('dialect is immutable.')626        super().__setattr__(attr, value)627 628 629class Excel(Dialect):630    """Describe the usual properties of Excel-generated CSV files."""631    delimiter = ','632    quotechar = '"'633    doublequote = True634    skipinitialspace = False635    lineterminator = '\r\n'636    quoting = QUOTE_MINIMAL637 638 639register_dialect("excel", Excel)640 641 642class ExcelTab(Excel):643    """Describe the usual properties of Excel-generated TAB-delimited files."""644    delimiter = '\t'645 646 647register_dialect("excel-tab", ExcelTab)648 649 650class UnixDialect(Dialect):651    """Describe the usual properties of Unix-generated CSV files."""652    delimiter = ','653    quotechar = '"'654    doublequote = True655    skipinitialspace = False656    lineterminator = '\n'657    quoting = QUOTE_ALL658 659 660register_dialect("unix", UnixDialect)661 662 663class DictReader():664    def __init__(self, f, fieldnames=None, restkey=None, restval=None,665                 *args, **kwds):666        self._fieldnames = fieldnames   # list of keys for the dict667        self.restkey = restkey          # key to catch long rows668        self.restval = restval          # default value for short rows669        self.dialect = kwds.get('dialect', "excel")670        self.reader = Reader(f, self.dialect, *args, **kwds)671        self.line_num = 0672 673    def __iter__(self):674        return self675 676    @property677    def fieldnames(self):678        if self._fieldnames is None:679            try:680                self._fieldnames = next(self.reader)681            except StopIteration:682                pass683        self.line_num = self.reader.line_num684        return self._fieldnames685 686    @fieldnames.setter687    def fieldnames(self, value):688        self._fieldnames = value689 690    def __next__(self):691        if self.line_num == 0:692            # Used only for its side effect.693            self.fieldnames694        row = next(self.reader)695        self.line_num = self.reader.line_num696 697        # unlike the basic reader, we prefer not to return blanks,698        # because we will typically wind up with a dict full of None699        # values700        while row == []:701            row = next(self.reader)702        d = dict(zip(self.fieldnames, row))703        lf = len(self.fieldnames)704        lr = len(row)705        if lf < lr:706            d[self.restkey] = row[lf:]707        elif lf > lr:708            for key in self.fieldnames[lr:]:709                d[key] = self.restval710        return d711 712    next = __next__713 714 715class DictWriter():716    def __init__(self, f, fieldnames, *args, **kwds):717        self.fieldnames = fieldnames    # list of keys for the dict718        self.extrasaction = kwds.get('extrasaction', "raise")719        self.restval = kwds.get('restval', "")  # for writing short dicts720        if self.extrasaction.lower() not in ("raise", "ignore"):721            raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'"722                             % self.extrasaction)723        dialect = kwds.get('dialect', "excel")724        self.writer = Writer(f, dialect, *args, **kwds)725 726    def writeheader(self):727        header = dict(zip(self.fieldnames, self.fieldnames))728        self.writerow(header)729 730    def _dict_to_list(self, rowdict):731        if self.extrasaction == "raise":732            wrong_fields = [k for k in rowdict if k not in self.fieldnames]733            if wrong_fields:734                raise ValueError("dict contains fields not in fieldnames: " +735                                 ", ".join([repr(x) for x in wrong_fields]))736        return (rowdict.get(key, self.restval) for key in self.fieldnames)737 738    def writerow(self, rowdict):739        return self.writer.writerow(self._dict_to_list(rowdict))740 741    def writerows(self, rowdicts):742        return self.writer.writerows(map(self._dict_to_list, rowdicts))743