codekingpro/portable-devtools

1# -*- coding: utf-8 -*-2#3# Copyright (C) 2013-2017 Vinay Sajip.4# Licensed to the Python Software Foundation under a contributor agreement.5# See LICENSE.txt and CONTRIBUTORS.txt.6#7from __future__ import absolute_import8 9import os10import re11import shutil12import sys13 14try:15    import ssl16except ImportError:  # pragma: no cover17    ssl = None18 19if sys.version_info[0] < 3:  # pragma: no cover20    from StringIO import StringIO21    string_types = basestring,22    text_type = unicode23    from types import FileType as file_type24    import __builtin__ as builtins25    import ConfigParser as configparser26    from urlparse import urlparse, urlunparse, urljoin, urlsplit, urlunsplit27    from urllib import (urlretrieve, quote as _quote, unquote, url2pathname,28                        pathname2url, ContentTooShortError, splittype)29 30    def quote(s):31        if isinstance(s, unicode):32            s = s.encode('utf-8')33        return _quote(s)34 35    import urllib236    from urllib2 import (Request, urlopen, URLError, HTTPError,37                         HTTPBasicAuthHandler, HTTPPasswordMgr, HTTPHandler,38                         HTTPRedirectHandler, build_opener)39    if ssl:40        from urllib2 import HTTPSHandler41    import httplib42    import xmlrpclib43    import Queue as queue44    from HTMLParser import HTMLParser45    import htmlentitydefs46    raw_input = raw_input47    from itertools import ifilter as filter48    from itertools import ifilterfalse as filterfalse49 50    # Leaving this around for now, in case it needs resurrecting in some way51    # _userprog = None52    # def splituser(host):53    # """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""54    # global _userprog55    # if _userprog is None:56    # import re57    # _userprog = re.compile('^(.*)@(.*)$')58 59    # match = _userprog.match(host)60    # if match: return match.group(1, 2)61    # return None, host62 63else:  # pragma: no cover64    from io import StringIO65    string_types = str,66    text_type = str67    from io import TextIOWrapper as file_type68    import builtins69    import configparser70    from urllib.parse import (urlparse, urlunparse, urljoin, quote, unquote,71                              urlsplit, urlunsplit, splittype)72    from urllib.request import (urlopen, urlretrieve, Request, url2pathname,73                                pathname2url, HTTPBasicAuthHandler,74                                HTTPPasswordMgr, HTTPHandler,75                                HTTPRedirectHandler, build_opener)76    if ssl:77        from urllib.request import HTTPSHandler78    from urllib.error import HTTPError, URLError, ContentTooShortError79    import http.client as httplib80    import urllib.request as urllib281    import xmlrpc.client as xmlrpclib82    import queue83    from html.parser import HTMLParser84    import html.entities as htmlentitydefs85    raw_input = input86    from itertools import filterfalse87    filter = filter88 89try:90    from ssl import match_hostname, CertificateError91except ImportError:  # pragma: no cover92 93    class CertificateError(ValueError):94        pass95 96    def _dnsname_match(dn, hostname, max_wildcards=1):97        """Matching according to RFC 6125, section 6.4.398 99        http://tools.ietf.org/html/rfc6125#section-6.4.3100        """101        pats = []102        if not dn:103            return False104 105        parts = dn.split('.')106        leftmost, remainder = parts[0], parts[1:]107 108        wildcards = leftmost.count('*')109        if wildcards > max_wildcards:110            # Issue #17980: avoid denials of service by refusing more111            # than one wildcard per fragment.  A survey of established112            # policy among SSL implementations showed it to be a113            # reasonable choice.114            raise CertificateError(115                "too many wildcards in certificate DNS name: " + repr(dn))116 117        # speed up common case w/o wildcards118        if not wildcards:119            return dn.lower() == hostname.lower()120 121        # RFC 6125, section 6.4.3, subitem 1.122        # The client SHOULD NOT attempt to match a presented identifier in which123        # the wildcard character comprises a label other than the left-most label.124        if leftmost == '*':125            # When '*' is a fragment by itself, it matches a non-empty dotless126            # fragment.127            pats.append('[^.]+')128        elif leftmost.startswith('xn--') or hostname.startswith('xn--'):129            # RFC 6125, section 6.4.3, subitem 3.130            # The client SHOULD NOT attempt to match a presented identifier131            # where the wildcard character is embedded within an A-label or132            # U-label of an internationalized domain name.133            pats.append(re.escape(leftmost))134        else:135            # Otherwise, '*' matches any dotless string, e.g. www*136            pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))137 138        # add the remaining fragments, ignore any wildcards139        for frag in remainder:140            pats.append(re.escape(frag))141 142        pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)143        return pat.match(hostname)144 145    def match_hostname(cert, hostname):146        """Verify that *cert* (in decoded format as returned by147        SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 and RFC 6125148        rules are followed, but IP addresses are not accepted for *hostname*.149 150        CertificateError is raised on failure. On success, the function151        returns nothing.152        """153        if not cert:154            raise ValueError("empty or no certificate, match_hostname needs a "155                             "SSL socket or SSL context with either "156                             "CERT_OPTIONAL or CERT_REQUIRED")157        dnsnames = []158        san = cert.get('subjectAltName', ())159        for key, value in san:160            if key == 'DNS':161                if _dnsname_match(value, hostname):162                    return163                dnsnames.append(value)164        if not dnsnames:165            # The subject is only checked when there is no dNSName entry166            # in subjectAltName167            for sub in cert.get('subject', ()):168                for key, value in sub:169                    # XXX according to RFC 2818, the most specific Common Name170                    # must be used.171                    if key == 'commonName':172                        if _dnsname_match(value, hostname):173                            return174                        dnsnames.append(value)175        if len(dnsnames) > 1:176            raise CertificateError("hostname %r "177                                   "doesn't match either of %s" %178                                   (hostname, ', '.join(map(repr, dnsnames))))179        elif len(dnsnames) == 1:180            raise CertificateError("hostname %r "181                                   "doesn't match %r" %182                                   (hostname, dnsnames[0]))183        else:184            raise CertificateError("no appropriate commonName or "185                                   "subjectAltName fields were found")186 187 188try:189    from types import SimpleNamespace as Container190except ImportError:  # pragma: no cover191 192    class Container(object):193        """194        A generic container for when multiple values need to be returned195        """196 197        def __init__(self, **kwargs):198            self.__dict__.update(kwargs)199 200 201try:202    from shutil import which203except ImportError:  # pragma: no cover204    # Implementation from Python 3.3205    def which(cmd, mode=os.F_OK | os.X_OK, path=None):206        """Given a command, mode, and a PATH string, return the path which207        conforms to the given mode on the PATH, or None if there is no such208        file.209 210        `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result211        of os.environ.get("PATH"), or can be overridden with a custom search212        path.213 214        """215 216        # Check that a given file can be accessed with the correct mode.217        # Additionally check that `file` is not a directory, as on Windows218        # directories pass the os.access check.219        def _access_check(fn, mode):220            return (os.path.exists(fn) and os.access(fn, mode)221                    and not os.path.isdir(fn))222 223        # If we're given a path with a directory part, look it up directly rather224        # than referring to PATH directories. This includes checking relative to the225        # current directory, e.g. ./script226        if os.path.dirname(cmd):227            if _access_check(cmd, mode):228                return cmd229            return None230 231        if path is None:232            path = os.environ.get("PATH", os.defpath)233        if not path:234            return None235        path = path.split(os.pathsep)236 237        if sys.platform == "win32":238            # The current directory takes precedence on Windows.239            if os.curdir not in path:240                path.insert(0, os.curdir)241 242            # PATHEXT is necessary to check on Windows.243            pathext = os.environ.get("PATHEXT", "").split(os.pathsep)244            # See if the given file matches any of the expected path extensions.245            # This will allow us to short circuit when given "python.exe".246            # If it does match, only test that one, otherwise we have to try247            # others.248            if any(cmd.lower().endswith(ext.lower()) for ext in pathext):249                files = [cmd]250            else:251                files = [cmd + ext for ext in pathext]252        else:253            # On other platforms you don't have things like PATHEXT to tell you254            # what file suffixes are executable, so just pass on cmd as-is.255            files = [cmd]256 257        seen = set()258        for dir in path:259            normdir = os.path.normcase(dir)260            if normdir not in seen:261                seen.add(normdir)262                for thefile in files:263                    name = os.path.join(dir, thefile)264                    if _access_check(name, mode):265                        return name266        return None267 268 269# ZipFile is a context manager in 2.7, but not in 2.6270 271from zipfile import ZipFile as BaseZipFile272 273if hasattr(BaseZipFile, '__enter__'):  # pragma: no cover274    ZipFile = BaseZipFile275else:  # pragma: no cover276    from zipfile import ZipExtFile as BaseZipExtFile277 278    class ZipExtFile(BaseZipExtFile):279 280        def __init__(self, base):281            self.__dict__.update(base.__dict__)282 283        def __enter__(self):284            return self285 286        def __exit__(self, *exc_info):287            self.close()288            # return None, so if an exception occurred, it will propagate289 290    class ZipFile(BaseZipFile):291 292        def __enter__(self):293            return self294 295        def __exit__(self, *exc_info):296            self.close()297            # return None, so if an exception occurred, it will propagate298 299        def open(self, *args, **kwargs):300            base = BaseZipFile.open(self, *args, **kwargs)301            return ZipExtFile(base)302 303 304try:305    from platform import python_implementation306except ImportError:  # pragma: no cover307 308    def python_implementation():309        """Return a string identifying the Python implementation."""310        if 'PyPy' in sys.version:311            return 'PyPy'312        if os.name == 'java':313            return 'Jython'314        if sys.version.startswith('IronPython'):315            return 'IronPython'316        return 'CPython'317 318 319import sysconfig320 321try:322    callable = callable323except NameError:  # pragma: no cover324    from collections.abc import Callable325 326    def callable(obj):327        return isinstance(obj, Callable)328 329 330try:331    fsencode = os.fsencode332    fsdecode = os.fsdecode333except AttributeError:  # pragma: no cover334    # Issue #99: on some systems (e.g. containerised),335    # sys.getfilesystemencoding() returns None, and we need a real value,336    # so fall back to utf-8. From the CPython 2.7 docs relating to Unix and337    # sys.getfilesystemencoding(): the return value is "the user’s preference338    # according to the result of nl_langinfo(CODESET), or None if the339    # nl_langinfo(CODESET) failed."340    _fsencoding = sys.getfilesystemencoding() or 'utf-8'341    if _fsencoding == 'mbcs':342        _fserrors = 'strict'343    else:344        _fserrors = 'surrogateescape'345 346    def fsencode(filename):347        if isinstance(filename, bytes):348            return filename349        elif isinstance(filename, text_type):350            return filename.encode(_fsencoding, _fserrors)351        else:352            raise TypeError("expect bytes or str, not %s" %353                            type(filename).__name__)354 355    def fsdecode(filename):356        if isinstance(filename, text_type):357            return filename358        elif isinstance(filename, bytes):359            return filename.decode(_fsencoding, _fserrors)360        else:361            raise TypeError("expect bytes or str, not %s" %362                            type(filename).__name__)363 364 365try:366    from tokenize import detect_encoding367except ImportError:  # pragma: no cover368    from codecs import BOM_UTF8, lookup369 370    cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)")371 372    def _get_normal_name(orig_enc):373        """Imitates get_normal_name in tokenizer.c."""374        # Only care about the first 12 characters.375        enc = orig_enc[:12].lower().replace("_", "-")376        if enc == "utf-8" or enc.startswith("utf-8-"):377            return "utf-8"378        if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \379           enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):380            return "iso-8859-1"381        return orig_enc382 383    def detect_encoding(readline):384        """385        The detect_encoding() function is used to detect the encoding that should386        be used to decode a Python source file.  It requires one argument, readline,387        in the same way as the tokenize() generator.388 389        It will call readline a maximum of twice, and return the encoding used390        (as a string) and a list of any lines (left as bytes) it has read in.391 392        It detects the encoding from the presence of a utf-8 bom or an encoding393        cookie as specified in pep-0263.  If both a bom and a cookie are present,394        but disagree, a SyntaxError will be raised.  If the encoding cookie is an395        invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,396        'utf-8-sig' is returned.397 398        If no encoding is specified, then the default of 'utf-8' will be returned.399        """400        try:401            filename = readline.__self__.name402        except AttributeError:403            filename = None404        bom_found = False405        encoding = None406        default = 'utf-8'407 408        def read_or_stop():409            try:410                return readline()411            except StopIteration:412                return b''413 414        def find_cookie(line):415            try:416                # Decode as UTF-8. Either the line is an encoding declaration,417                # in which case it should be pure ASCII, or it must be UTF-8418                # per default encoding.419                line_string = line.decode('utf-8')420            except UnicodeDecodeError:421                msg = "invalid or missing encoding declaration"422                if filename is not None:423                    msg = '{} for {!r}'.format(msg, filename)424                raise SyntaxError(msg)425 426            matches = cookie_re.findall(line_string)427            if not matches:428                return None429            encoding = _get_normal_name(matches[0])430            try:431                codec = lookup(encoding)432            except LookupError:433                # This behaviour mimics the Python interpreter434                if filename is None:435                    msg = "unknown encoding: " + encoding436                else:437                    msg = "unknown encoding for {!r}: {}".format(438                        filename, encoding)439                raise SyntaxError(msg)440 441            if bom_found:442                if codec.name != 'utf-8':443                    # This behaviour mimics the Python interpreter444                    if filename is None:445                        msg = 'encoding problem: utf-8'446                    else:447                        msg = 'encoding problem for {!r}: utf-8'.format(448                            filename)449                    raise SyntaxError(msg)450                encoding += '-sig'451            return encoding452 453        first = read_or_stop()454        if first.startswith(BOM_UTF8):455            bom_found = True456            first = first[3:]457            default = 'utf-8-sig'458        if not first:459            return default, []460 461        encoding = find_cookie(first)462        if encoding:463            return encoding, [first]464 465        second = read_or_stop()466        if not second:467            return default, [first]468 469        encoding = find_cookie(second)470        if encoding:471            return encoding, [first, second]472 473        return default, [first, second]474 475 476# For converting & <-> &amp; etc.477try:478    from html import escape479except ImportError:480    from cgi import escape481if sys.version_info[:2] < (3, 4):482    unescape = HTMLParser().unescape483else:484    from html import unescape485 486try:487    from collections import ChainMap488except ImportError:  # pragma: no cover489    from collections import MutableMapping490 491    try:492        from reprlib import recursive_repr as _recursive_repr493    except ImportError:494 495        def _recursive_repr(fillvalue='...'):496            '''497            Decorator to make a repr function return fillvalue for a recursive498            call499            '''500 501            def decorating_function(user_function):502                repr_running = set()503 504                def wrapper(self):505                    key = id(self), get_ident()506                    if key in repr_running:507                        return fillvalue508                    repr_running.add(key)509                    try:510                        result = user_function(self)511                    finally:512                        repr_running.discard(key)513                    return result514 515                # Can't use functools.wraps() here because of bootstrap issues516                wrapper.__module__ = getattr(user_function, '__module__')517                wrapper.__doc__ = getattr(user_function, '__doc__')518                wrapper.__name__ = getattr(user_function, '__name__')519                wrapper.__annotations__ = getattr(user_function,520                                                  '__annotations__', {})521                return wrapper522 523            return decorating_function524 525    class ChainMap(MutableMapping):526        '''527        A ChainMap groups multiple dicts (or other mappings) together528        to create a single, updateable view.529 530        The underlying mappings are stored in a list.  That list is public and can531        accessed or updated using the *maps* attribute.  There is no other state.532 533        Lookups search the underlying mappings successively until a key is found.534        In contrast, writes, updates, and deletions only operate on the first535        mapping.536        '''537 538        def __init__(self, *maps):539            '''Initialize a ChainMap by setting *maps* to the given mappings.540            If no mappings are provided, a single empty dictionary is used.541 542            '''543            self.maps = list(maps) or [{}]  # always at least one map544 545        def __missing__(self, key):546            raise KeyError(key)547 548        def __getitem__(self, key):549            for mapping in self.maps:550                try:551                    return mapping[552                        key]  # can't use 'key in mapping' with defaultdict553                except KeyError:554                    pass555            return self.__missing__(556                key)  # support subclasses that define __missing__557 558        def get(self, key, default=None):559            return self[key] if key in self else default560 561        def __len__(self):562            return len(set().union(563                *self.maps))  # reuses stored hash values if possible564 565        def __iter__(self):566            return iter(set().union(*self.maps))567 568        def __contains__(self, key):569            return any(key in m for m in self.maps)570 571        def __bool__(self):572            return any(self.maps)573 574        @_recursive_repr()575        def __repr__(self):576            return '{0.__class__.__name__}({1})'.format(577                self, ', '.join(map(repr, self.maps)))578 579        @classmethod580        def fromkeys(cls, iterable, *args):581            'Create a ChainMap with a single dict created from the iterable.'582            return cls(dict.fromkeys(iterable, *args))583 584        def copy(self):585            'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'586            return self.__class__(self.maps[0].copy(), *self.maps[1:])587 588        __copy__ = copy589 590        def new_child(self):  # like Django's Context.push()591            'New ChainMap with a new dict followed by all previous maps.'592            return self.__class__({}, *self.maps)593 594        @property595        def parents(self):  # like Django's Context.pop()596            'New ChainMap from maps[1:].'597            return self.__class__(*self.maps[1:])598 599        def __setitem__(self, key, value):600            self.maps[0][key] = value601 602        def __delitem__(self, key):603            try:604                del self.maps[0][key]605            except KeyError:606                raise KeyError(607                    'Key not found in the first mapping: {!r}'.format(key))608 609        def popitem(self):610            'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.'611            try:612                return self.maps[0].popitem()613            except KeyError:614                raise KeyError('No keys found in the first mapping.')615 616        def pop(self, key, *args):617            'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].'618            try:619                return self.maps[0].pop(key, *args)620            except KeyError:621                raise KeyError(622                    'Key not found in the first mapping: {!r}'.format(key))623 624        def clear(self):625            'Clear maps[0], leaving maps[1:] intact.'626            self.maps[0].clear()627 628 629try:630    from importlib.util import cache_from_source  # Python >= 3.4631except ImportError:  # pragma: no cover632 633    def cache_from_source(path, debug_override=None):634        assert path.endswith('.py')635        if debug_override is None:636            debug_override = __debug__637        if debug_override:638            suffix = 'c'639        else:640            suffix = 'o'641        return path + suffix642 643 644try:645    from collections import OrderedDict646except ImportError:  # pragma: no cover647    # {{{ http://code.activestate.com/recipes/576693/ (r9)648    # Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.649    # Passes Python2.7's test suite and incorporates all the latest updates.650    try:651        from thread import get_ident as _get_ident652    except ImportError:653        from dummy_thread import get_ident as _get_ident654 655    try:656        from _abcoll import KeysView, ValuesView, ItemsView657    except ImportError:658        pass659 660    class OrderedDict(dict):661        'Dictionary that remembers insertion order'662 663        # An inherited dict maps keys to values.664        # The inherited dict provides __getitem__, __len__, __contains__, and get.665        # The remaining methods are order-aware.666        # Big-O running times for all methods are the same as for regular dictionaries.667 668        # The internal self.__map dictionary maps keys to links in a doubly linked list.669        # The circular doubly linked list starts and ends with a sentinel element.670        # The sentinel element never gets deleted (this simplifies the algorithm).671        # Each link is stored as a list of length three:  [PREV, NEXT, KEY].672 673        def __init__(self, *args, **kwds):674            '''Initialize an ordered dictionary.  Signature is the same as for675            regular dictionaries, but keyword arguments are not recommended676            because their insertion order is arbitrary.677 678            '''679            if len(args) > 1:680                raise TypeError('expected at most 1 arguments, got %d' %681                                len(args))682            try:683                self.__root684            except AttributeError:685                self.__root = root = []  # sentinel node686                root[:] = [root, root, None]687                self.__map = {}688            self.__update(*args, **kwds)689 690        def __setitem__(self, key, value, dict_setitem=dict.__setitem__):691            'od.__setitem__(i, y) <==> od[i]=y'692            # Setting a new item creates a new link which goes at the end of the linked693            # list, and the inherited dictionary is updated with the new key/value pair.694            if key not in self:695                root = self.__root696                last = root[0]697                last[1] = root[0] = self.__map[key] = [last, root, key]698            dict_setitem(self, key, value)699 700        def __delitem__(self, key, dict_delitem=dict.__delitem__):701            'od.__delitem__(y) <==> del od[y]'702            # Deleting an existing item uses self.__map to find the link which is703            # then removed by updating the links in the predecessor and successor nodes.704            dict_delitem(self, key)705            link_prev, link_next, key = self.__map.pop(key)706            link_prev[1] = link_next707            link_next[0] = link_prev708 709        def __iter__(self):710            'od.__iter__() <==> iter(od)'711            root = self.__root712            curr = root[1]713            while curr is not root:714                yield curr[2]715                curr = curr[1]716 717        def __reversed__(self):718            'od.__reversed__() <==> reversed(od)'719            root = self.__root720            curr = root[0]721            while curr is not root:722                yield curr[2]723                curr = curr[0]724 725        def clear(self):726            'od.clear() -> None.  Remove all items from od.'727            try:728                for node in self.__map.itervalues():729                    del node[:]730                root = self.__root731                root[:] = [root, root, None]732                self.__map.clear()733            except AttributeError:734                pass735            dict.clear(self)736 737        def popitem(self, last=True):738            '''od.popitem() -> (k, v), return and remove a (key, value) pair.739            Pairs are returned in LIFO order if last is true or FIFO order if false.740 741            '''742            if not self:743                raise KeyError('dictionary is empty')744            root = self.__root745            if last:746                link = root[0]747                link_prev = link[0]748                link_prev[1] = root749                root[0] = link_prev750            else:751                link = root[1]752                link_next = link[1]753                root[1] = link_next754                link_next[0] = root755            key = link[2]756            del self.__map[key]757            value = dict.pop(self, key)758            return key, value759 760        # -- the following methods do not depend on the internal structure --761 762        def keys(self):763            'od.keys() -> list of keys in od'764            return list(self)765 766        def values(self):767            'od.values() -> list of values in od'768            return [self[key] for key in self]769 770        def items(self):771            'od.items() -> list of (key, value) pairs in od'772            return [(key, self[key]) for key in self]773 774        def iterkeys(self):775            'od.iterkeys() -> an iterator over the keys in od'776            return iter(self)777 778        def itervalues(self):779            'od.itervalues -> an iterator over the values in od'780            for k in self:781                yield self[k]782 783        def iteritems(self):784            'od.iteritems -> an iterator over the (key, value) items in od'785            for k in self:786                yield (k, self[k])787 788        def update(*args, **kwds):789            '''od.update(E, **F) -> None.  Update od from dict/iterable E and F.790 791            If E is a dict instance, does:           for k in E: od[k] = E[k]792            If E has a .keys() method, does:         for k in E.keys(): od[k] = E[k]793            Or if E is an iterable of items, does:   for k, v in E: od[k] = v794            In either case, this is followed by:     for k, v in F.items(): od[k] = v795 796            '''797            if len(args) > 2:798                raise TypeError('update() takes at most 2 positional '799                                'arguments (%d given)' % (len(args), ))800            elif not args:801                raise TypeError('update() takes at least 1 argument (0 given)')802            self = args[0]803            # Make progressively weaker assumptions about "other"804            other = ()805            if len(args) == 2:806                other = args[1]807            if isinstance(other, dict):808                for key in other:809                    self[key] = other[key]810            elif hasattr(other, 'keys'):811                for key in other.keys():812                    self[key] = other[key]813            else:814                for key, value in other:815                    self[key] = value816            for key, value in kwds.items():817                self[key] = value818 819        __update = update  # let subclasses override update without breaking __init__820 821        __marker = object()822 823        def pop(self, key, default=__marker):824            '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.825            If key is not found, d is returned if given, otherwise KeyError is raised.826 827            '''828            if key in self:829                result = self[key]830                del self[key]831                return result832            if default is self.__marker:833                raise KeyError(key)834            return default835 836        def setdefault(self, key, default=None):837            'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'838            if key in self:839                return self[key]840            self[key] = default841            return default842 843        def __repr__(self, _repr_running=None):844            'od.__repr__() <==> repr(od)'845            if not _repr_running:846                _repr_running = {}847            call_key = id(self), _get_ident()848            if call_key in _repr_running:849                return '...'850            _repr_running[call_key] = 1851            try:852                if not self:853                    return '%s()' % (self.__class__.__name__, )854                return '%s(%r)' % (self.__class__.__name__, self.items())855            finally:856                del _repr_running[call_key]857 858        def __reduce__(self):859            'Return state information for pickling'860            items = [[k, self[k]] for k in self]861            inst_dict = vars(self).copy()862            for k in vars(OrderedDict()):863                inst_dict.pop(k, None)864            if inst_dict:865                return (self.__class__, (items, ), inst_dict)866            return self.__class__, (items, )867 868        def copy(self):869            'od.copy() -> a shallow copy of od'870            return self.__class__(self)871 872        @classmethod873        def fromkeys(cls, iterable, value=None):874            '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S875            and values equal to v (which defaults to None).876 877            '''878            d = cls()879            for key in iterable:880                d[key] = value881            return d882 883        def __eq__(self, other):884            '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive885            while comparison to a regular mapping is order-insensitive.886 887            '''888            if isinstance(other, OrderedDict):889                return len(self) == len(890                    other) and self.items() == other.items()891            return dict.__eq__(self, other)892 893        def __ne__(self, other):894            return not self == other895 896        # -- the following methods are only used in Python 2.7 --897 898        def viewkeys(self):899            "od.viewkeys() -> a set-like object providing a view on od's keys"900            return KeysView(self)901 902        def viewvalues(self):903            "od.viewvalues() -> an object providing a view on od's values"904            return ValuesView(self)905 906        def viewitems(self):907            "od.viewitems() -> a set-like object providing a view on od's items"908            return ItemsView(self)909 910 911try:912    from logging.config import BaseConfigurator, valid_ident913except ImportError:  # pragma: no cover914    IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I)915 916    def valid_ident(s):917        m = IDENTIFIER.match(s)918        if not m:919            raise ValueError('Not a valid Python identifier: %r' % s)920        return True921 922    # The ConvertingXXX classes are wrappers around standard Python containers,923    # and they serve to convert any suitable values in the container. The924    # conversion converts base dicts, lists and tuples to their wrapped925    # equivalents, whereas strings which match a conversion format are converted926    # appropriately.927    #928    # Each wrapper should have a configurator attribute holding the actual929    # configurator to use for conversion.930 931    class ConvertingDict(dict):932        """A converting dictionary wrapper."""933 934        def __getitem__(self, key):935            value = dict.__getitem__(self, key)936            result = self.configurator.convert(value)937            # If the converted value is different, save for next time938            if value is not result:939                self[key] = result940                if type(result) in (ConvertingDict, ConvertingList,941                                    ConvertingTuple):942                    result.parent = self943                    result.key = key944            return result945 946        def get(self, key, default=None):947            value = dict.get(self, key, default)948            result = self.configurator.convert(value)949            # If the converted value is different, save for next time950            if value is not result:951                self[key] = result952                if type(result) in (ConvertingDict, ConvertingList,953                                    ConvertingTuple):954                    result.parent = self955                    result.key = key956            return result957 958    def pop(self, key, default=None):959        value = dict.pop(self, key, default)960        result = self.configurator.convert(value)961        if value is not result:962            if type(result) in (ConvertingDict, ConvertingList,963                                ConvertingTuple):964                result.parent = self965                result.key = key966        return result967 968    class ConvertingList(list):969        """A converting list wrapper."""970 971        def __getitem__(self, key):972            value = list.__getitem__(self, key)973            result = self.configurator.convert(value)974            # If the converted value is different, save for next time975            if value is not result:976                self[key] = result977                if type(result) in (ConvertingDict, ConvertingList,978                                    ConvertingTuple):979                    result.parent = self980                    result.key = key981            return result982 983        def pop(self, idx=-1):984            value = list.pop(self, idx)985            result = self.configurator.convert(value)986            if value is not result:987                if type(result) in (ConvertingDict, ConvertingList,988                                    ConvertingTuple):989                    result.parent = self990            return result991 992    class ConvertingTuple(tuple):993        """A converting tuple wrapper."""994 995        def __getitem__(self, key):996            value = tuple.__getitem__(self, key)997            result = self.configurator.convert(value)998            if value is not result:999                if type(result) in (ConvertingDict, ConvertingList,1000                                    ConvertingTuple):1001                    result.parent = self1002                    result.key = key1003            return result1004 1005    class BaseConfigurator(object):1006        """1007        The configurator base class which defines some useful defaults.1008        """1009 1010        CONVERT_PATTERN = re.compile(r'^(?P<prefix>[a-z]+)://(?P<suffix>.*)$')1011 1012        WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')1013        DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')1014        INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')1015        DIGIT_PATTERN = re.compile(r'^\d+$')1016 1017        value_converters = {1018            'ext': 'ext_convert',1019            'cfg': 'cfg_convert',1020        }1021 1022        # We might want to use a different one, e.g. importlib1023        importer = staticmethod(__import__)1024 1025        def __init__(self, config):1026            self.config = ConvertingDict(config)1027            self.config.configurator = self1028 1029        def resolve(self, s):1030            """1031            Resolve strings to objects using standard import and attribute1032            syntax.1033            """1034            name = s.split('.')1035            used = name.pop(0)1036            try:1037                found = self.importer(used)1038                for frag in name:1039                    used += '.' + frag1040                    try:1041                        found = getattr(found, frag)1042                    except AttributeError:1043                        self.importer(used)1044                        found = getattr(found, frag)1045                return found1046            except ImportError:1047                e, tb = sys.exc_info()[1:]1048                v = ValueError('Cannot resolve %r: %s' % (s, e))1049                v.__cause__, v.__traceback__ = e, tb1050                raise v1051 1052        def ext_convert(self, value):1053            """Default converter for the ext:// protocol."""1054            return self.resolve(value)1055 1056        def cfg_convert(self, value):1057            """Default converter for the cfg:// protocol."""1058            rest = value1059            m = self.WORD_PATTERN.match(rest)1060            if m is None:1061                raise ValueError("Unable to convert %r" % value)1062            else:1063                rest = rest[m.end():]1064                d = self.config[m.groups()[0]]1065                while rest:1066                    m = self.DOT_PATTERN.match(rest)1067                    if m:1068                        d = d[m.groups()[0]]1069                    else:1070                        m = self.INDEX_PATTERN.match(rest)1071                        if m:1072                            idx = m.groups()[0]1073                            if not self.DIGIT_PATTERN.match(idx):1074                                d = d[idx]1075                            else:1076                                try:1077                                    n = int(1078                                        idx1079                                    )  # try as number first (most likely)1080                                    d = d[n]1081                                except TypeError:1082                                    d = d[idx]1083                    if m:1084                        rest = rest[m.end():]1085                    else:1086                        raise ValueError('Unable to convert '1087                                         '%r at %r' % (value, rest))1088            # rest should be empty1089            return d1090 1091        def convert(self, value):1092            """1093            Convert values to an appropriate type. dicts, lists and tuples are1094            replaced by their converting alternatives. Strings are checked to1095            see if they have a conversion format and are converted if they do.1096            """1097            if not isinstance(value, ConvertingDict) and isinstance(1098                    value, dict):1099                value = ConvertingDict(value)1100                value.configurator = self1101            elif not isinstance(value, ConvertingList) and isinstance(1102                    value, list):1103                value = ConvertingList(value)1104                value.configurator = self1105            elif not isinstance(value, ConvertingTuple) and isinstance(value, tuple):1106                value = ConvertingTuple(value)1107                value.configurator = self1108            elif isinstance(value, string_types):1109                m = self.CONVERT_PATTERN.match(value)1110                if m:1111                    d = m.groupdict()1112                    prefix = d['prefix']1113                    converter = self.value_converters.get(prefix, None)1114                    if converter:1115                        suffix = d['suffix']1116                        converter = getattr(self, converter)1117                        value = converter(suffix)1118            return value1119 1120        def configure_custom(self, config):1121            """Configure an object with a user-supplied factory."""1122            c = config.pop('()')1123            if not callable(c):1124                c = self.resolve(c)1125            props = config.pop('.', None)1126            # Check for valid identifiers1127            kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])1128            result = c(**kwargs)1129            if props:1130                for name, value in props.items():1131                    setattr(result, name, value)1132            return result1133 1134        def as_tuple(self, value):1135            """Utility function which converts lists to tuples."""1136            if isinstance(value, list):1137                value = tuple(value)1138            return value1139