codekingpro/portable-devtools

1import logging2import os3import os.path4import socket5import sys6import warnings7from base64 import b64encode8 9from urllib3 import PoolManager, Timeout, proxy_from_url10from urllib3.exceptions import (11    ConnectTimeoutError as URLLib3ConnectTimeoutError,12)13from urllib3.exceptions import (14    LocationParseError,15    NewConnectionError,16    ProtocolError,17    ProxyError,18)19from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError20from urllib3.exceptions import SSLError as URLLib3SSLError21from urllib3.util.retry import Retry22from urllib3.util.ssl_ import (23    OP_NO_COMPRESSION,24    PROTOCOL_TLS,25    OP_NO_SSLv2,26    OP_NO_SSLv3,27    is_ipaddress,28    ssl,29)30from urllib3.util.url import parse_url31 32try:33    from urllib3.util.ssl_ import OP_NO_TICKET, PROTOCOL_TLS_CLIENT34except ImportError:35    # Fallback directly to ssl for version of urllib3 before 1.26.36    # They are available in the standard library starting in Python 3.6.37    from ssl import OP_NO_TICKET, PROTOCOL_TLS_CLIENT38 39try:40    # pyopenssl will be removed in urllib3 2.0, we'll fall back to ssl_ at that point.41    # This can be removed once our urllib3 floor is raised to >= 2.0.42    with warnings.catch_warnings():43        warnings.simplefilter("ignore", category=DeprecationWarning)44        # Always import the original SSLContext, even if it has been patched45        from urllib3.contrib.pyopenssl import (46            orig_util_SSLContext as SSLContext,47        )48except ImportError:49    from urllib3.util.ssl_ import SSLContext50 51try:52    from urllib3.util.ssl_ import DEFAULT_CIPHERS53except ImportError:54    # Defer to system configuration starting with55    # urllib3 2.0. This will choose the ciphers provided by56    # Openssl 1.1.1+ or secure system defaults.57    DEFAULT_CIPHERS = None58 59import botocore.awsrequest60from botocore.compat import (61    IPV6_ADDRZ_RE,62    ensure_bytes,63    filter_ssl_warnings,64    unquote,65    urlparse,66)67from botocore.exceptions import (68    ConnectionClosedError,69    ConnectTimeoutError,70    EndpointConnectionError,71    HTTPClientError,72    InvalidProxiesConfigError,73    ProxyConnectionError,74    ReadTimeoutError,75    SSLError,76)77 78filter_ssl_warnings()79logger = logging.getLogger(__name__)80DEFAULT_TIMEOUT = 6081MAX_POOL_CONNECTIONS = 1082DEFAULT_CA_BUNDLE = os.path.join(os.path.dirname(__file__), 'cacert.pem')83 84try:85    from certifi import where86except ImportError:87 88    def where():89        return DEFAULT_CA_BUNDLE90 91 92def get_cert_path(verify):93    if verify is not True:94        return verify95 96    cert_path = where()97    logger.debug(f"Certificate path: {cert_path}")98 99    return cert_path100 101 102def create_urllib3_context(103    ssl_version=None, cert_reqs=None, options=None, ciphers=None104):105    """This function is a vendored version of the same function in urllib3106 107    We vendor this function to ensure that the SSL contexts we construct108    always use the std lib SSLContext instead of pyopenssl.109    """110    # PROTOCOL_TLS is deprecated in Python 3.10111    if not ssl_version or ssl_version == PROTOCOL_TLS:112        ssl_version = PROTOCOL_TLS_CLIENT113 114    context = SSLContext(ssl_version)115 116    if ciphers:117        context.set_ciphers(ciphers)118    elif DEFAULT_CIPHERS:119        context.set_ciphers(DEFAULT_CIPHERS)120 121    # Setting the default here, as we may have no ssl module on import122    cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs123 124    if options is None:125        options = 0126        # SSLv2 is easily broken and is considered harmful and dangerous127        options |= OP_NO_SSLv2128        # SSLv3 has several problems and is now dangerous129        options |= OP_NO_SSLv3130        # Disable compression to prevent CRIME attacks for OpenSSL 1.0+131        # (issue urllib3#309)132        options |= OP_NO_COMPRESSION133        # TLSv1.2 only. Unless set explicitly, do not request tickets.134        # This may save some bandwidth on wire, and although the ticket is encrypted,135        # there is a risk associated with it being on wire,136        # if the server is not rotating its ticketing keys properly.137        options |= OP_NO_TICKET138 139    context.options |= options140 141    # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is142    # necessary for conditional client cert authentication with TLS 1.3.143    # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older144    # versions of Python.  We only enable on Python 3.7.4+ or if certificate145    # verification is enabled to work around Python issue #37428146    # See: https://bugs.python.org/issue37428147    if (148        cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)149    ) and getattr(context, "post_handshake_auth", None) is not None:150        context.post_handshake_auth = True151 152    def disable_check_hostname():153        if (154            getattr(context, "check_hostname", None) is not None155        ):  # Platform-specific: Python 3.2156            # We do our own verification, including fingerprints and alternative157            # hostnames. So disable it here158            context.check_hostname = False159 160    # The order of the below lines setting verify_mode and check_hostname161    # matter due to safe-guards SSLContext has to prevent an SSLContext with162    # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more163    # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used164    # or not so we don't know the initial state of the freshly created SSLContext.165    if cert_reqs == ssl.CERT_REQUIRED:166        context.verify_mode = cert_reqs167        disable_check_hostname()168    else:169        disable_check_hostname()170        context.verify_mode = cert_reqs171 172    # Enable logging of TLS session keys via defacto standard environment variable173    # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values.174    if hasattr(context, "keylog_filename"):175        sslkeylogfile = os.environ.get("SSLKEYLOGFILE")176        if sslkeylogfile and not sys.flags.ignore_environment:177            context.keylog_filename = sslkeylogfile178 179    return context180 181 182def ensure_boolean(val):183    """Ensures a boolean value if a string or boolean is provided184 185    For strings, the value for True/False is case insensitive186    """187    if isinstance(val, bool):188        return val189    else:190        return val.lower() == 'true'191 192 193def mask_proxy_url(proxy_url):194    """195    Mask proxy url credentials.196 197    :type proxy_url: str198    :param proxy_url: The proxy url, i.e. https://username:password@proxy.com199 200    :return: Masked proxy url, i.e. https://***:***@proxy.com201    """202    mask = '*' * 3203    parsed_url = urlparse(proxy_url)204    if parsed_url.username:205        proxy_url = proxy_url.replace(parsed_url.username, mask, 1)206    if parsed_url.password:207        proxy_url = proxy_url.replace(parsed_url.password, mask, 1)208    return proxy_url209 210 211def _is_ipaddress(host):212    """Wrap urllib3's is_ipaddress to support bracketed IPv6 addresses."""213    return is_ipaddress(host) or bool(IPV6_ADDRZ_RE.match(host))214 215 216class ProxyConfiguration:217    """Represents a proxy configuration dictionary and additional settings.218 219    This class represents a proxy configuration dictionary and provides utility220    functions to retrieve well structured proxy urls and proxy headers from the221    proxy configuration dictionary.222    """223 224    def __init__(self, proxies=None, proxies_settings=None):225        if proxies is None:226            proxies = {}227        if proxies_settings is None:228            proxies_settings = {}229 230        self._proxies = proxies231        self._proxies_settings = proxies_settings232 233    def proxy_url_for(self, url):234        """Retrieves the corresponding proxy url for a given url."""235        parsed_url = urlparse(url)236        proxy = self._proxies.get(parsed_url.scheme)237        if proxy:238            proxy = self._fix_proxy_url(proxy)239        return proxy240 241    def proxy_headers_for(self, proxy_url):242        """Retrieves the corresponding proxy headers for a given proxy url."""243        headers = {}244        username, password = self._get_auth_from_url(proxy_url)245        if username and password:246            basic_auth = self._construct_basic_auth(username, password)247            headers['Proxy-Authorization'] = basic_auth248        return headers249 250    @property251    def settings(self):252        return self._proxies_settings253 254    def _fix_proxy_url(self, proxy_url):255        if proxy_url.startswith('http:') or proxy_url.startswith('https:'):256            return proxy_url257        elif proxy_url.startswith('//'):258            return 'http:' + proxy_url259        else:260            return 'http://' + proxy_url261 262    def _construct_basic_auth(self, username, password):263        auth_str = f'{username}:{password}'264        encoded_str = b64encode(auth_str.encode('ascii')).strip().decode()265        return f'Basic {encoded_str}'266 267    def _get_auth_from_url(self, url):268        parsed_url = urlparse(url)269        try:270            return unquote(parsed_url.username), unquote(parsed_url.password)271        except (AttributeError, TypeError):272            return None, None273 274 275class URLLib3Session:276    """A basic HTTP client that supports connection pooling and proxies.277 278    This class is inspired by requests.adapters.HTTPAdapter, but has been279    boiled down to meet the use cases needed by botocore. For the most part280    this classes matches the functionality of HTTPAdapter in requests v2.7.0281    (the same as our vendored version). The only major difference of note is282    that we currently do not support sending chunked requests. While requests283    v2.7.0 implemented this themselves, later version urllib3 support this284    directly via a flag to urlopen so enabling it if needed should be trivial.285    """286 287    def __init__(288        self,289        verify=True,290        proxies=None,291        timeout=None,292        max_pool_connections=MAX_POOL_CONNECTIONS,293        socket_options=None,294        client_cert=None,295        proxies_config=None,296    ):297        self._verify = verify298        self._proxy_config = ProxyConfiguration(299            proxies=proxies, proxies_settings=proxies_config300        )301        self._pool_classes_by_scheme = {302            'http': botocore.awsrequest.AWSHTTPConnectionPool,303            'https': botocore.awsrequest.AWSHTTPSConnectionPool,304        }305        if timeout is None:306            timeout = DEFAULT_TIMEOUT307        if not isinstance(timeout, (int, float)):308            timeout = Timeout(connect=timeout[0], read=timeout[1])309 310        self._cert_file = None311        self._key_file = None312        if isinstance(client_cert, str):313            self._cert_file = client_cert314        elif isinstance(client_cert, tuple):315            self._cert_file, self._key_file = client_cert316 317        self._timeout = timeout318        self._max_pool_connections = max_pool_connections319        self._socket_options = socket_options320        if socket_options is None:321            self._socket_options = []322        self._proxy_managers = {}323        self._manager = PoolManager(**self._get_pool_manager_kwargs())324        self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme325 326    def _proxies_kwargs(self, **kwargs):327        proxies_settings = self._proxy_config.settings328        proxies_kwargs = {329            'use_forwarding_for_https': proxies_settings.get(330                'proxy_use_forwarding_for_https'331            ),332            **kwargs,333        }334        return {k: v for k, v in proxies_kwargs.items() if v is not None}335 336    def _get_pool_manager_kwargs(self, **extra_kwargs):337        pool_manager_kwargs = {338            'timeout': self._timeout,339            'maxsize': self._max_pool_connections,340            'ssl_context': self._get_ssl_context(),341            'socket_options': self._socket_options,342            'cert_file': self._cert_file,343            'key_file': self._key_file,344        }345        pool_manager_kwargs.update(**extra_kwargs)346        return pool_manager_kwargs347 348    def _get_ssl_context(self):349        return create_urllib3_context()350 351    def _get_proxy_manager(self, proxy_url):352        if proxy_url not in self._proxy_managers:353            proxy_headers = self._proxy_config.proxy_headers_for(proxy_url)354            proxy_ssl_context = self._setup_proxy_ssl_context(proxy_url)355            proxy_manager_kwargs = self._get_pool_manager_kwargs(356                proxy_headers=proxy_headers357            )358            proxy_manager_kwargs.update(359                self._proxies_kwargs(proxy_ssl_context=proxy_ssl_context)360            )361            proxy_manager = proxy_from_url(proxy_url, **proxy_manager_kwargs)362            proxy_manager.pool_classes_by_scheme = self._pool_classes_by_scheme363            self._proxy_managers[proxy_url] = proxy_manager364 365        return self._proxy_managers[proxy_url]366 367    def _path_url(self, url):368        parsed_url = urlparse(url)369        path = parsed_url.path370        if not path:371            path = '/'372        if parsed_url.query:373            path = path + '?' + parsed_url.query374        return path375 376    def _setup_ssl_cert(self, conn, url, verify):377        if url.lower().startswith('https') and verify:378            conn.cert_reqs = 'CERT_REQUIRED'379            conn.ca_certs = get_cert_path(verify)380        else:381            conn.cert_reqs = 'CERT_NONE'382            conn.ca_certs = None383 384    def _setup_proxy_ssl_context(self, proxy_url):385        proxies_settings = self._proxy_config.settings386        proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle')387        proxy_cert = proxies_settings.get('proxy_client_cert')388        if proxy_ca_bundle is None and proxy_cert is None:389            return None390 391        context = self._get_ssl_context()392        try:393            url = parse_url(proxy_url)394            # urllib3 disables this by default but we need it for proper395            # proxy tls negotiation when proxy_url is not an IP Address396            if not _is_ipaddress(url.host):397                context.check_hostname = True398            if proxy_ca_bundle is not None:399                context.load_verify_locations(cafile=proxy_ca_bundle)400 401            if isinstance(proxy_cert, tuple):402                context.load_cert_chain(proxy_cert[0], keyfile=proxy_cert[1])403            elif isinstance(proxy_cert, str):404                context.load_cert_chain(proxy_cert)405 406            return context407        except (OSError, URLLib3SSLError, LocationParseError) as e:408            raise InvalidProxiesConfigError(error=e)409 410    def _get_connection_manager(self, url, proxy_url=None):411        if proxy_url:412            manager = self._get_proxy_manager(proxy_url)413        else:414            manager = self._manager415        return manager416 417    def _get_request_target(self, url, proxy_url):418        has_proxy = proxy_url is not None419 420        if not has_proxy:421            return self._path_url(url)422 423        # HTTP proxies expect the request_target to be the absolute url to know424        # which host to establish a connection to. urllib3 also supports425        # forwarding for HTTPS through the 'use_forwarding_for_https' parameter.426        proxy_scheme = urlparse(proxy_url).scheme427        using_https_forwarding_proxy = (428            proxy_scheme == 'https'429            and self._proxies_kwargs().get('use_forwarding_for_https', False)430        )431 432        if using_https_forwarding_proxy or url.startswith('http:'):433            return url434        else:435            return self._path_url(url)436 437    def _chunked(self, headers):438        transfer_encoding = headers.get('Transfer-Encoding', b'')439        transfer_encoding = ensure_bytes(transfer_encoding)440        return transfer_encoding.lower() == b'chunked'441 442    def close(self):443        self._manager.clear()444        for manager in self._proxy_managers.values():445            manager.clear()446 447    def send(self, request):448        try:449            proxy_url = self._proxy_config.proxy_url_for(request.url)450            manager = self._get_connection_manager(request.url, proxy_url)451            conn = manager.connection_from_url(request.url)452            self._setup_ssl_cert(conn, request.url, self._verify)453            if ensure_boolean(454                os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '')455            ):456                # This is currently an "experimental" feature which provides457                # no guarantees of backwards compatibility. It may be subject458                # to change or removal in any patch version. Anyone opting in459                # to this feature should strictly pin botocore.460                host = urlparse(request.url).hostname461                conn.proxy_headers['host'] = host462 463            request_target = self._get_request_target(request.url, proxy_url)464            urllib_response = conn.urlopen(465                method=request.method,466                url=request_target,467                body=request.body,468                headers=request.headers,469                retries=Retry(False),470                assert_same_host=False,471                preload_content=False,472                decode_content=False,473                chunked=self._chunked(request.headers),474            )475 476            http_response = botocore.awsrequest.AWSResponse(477                request.url,478                urllib_response.status,479                urllib_response.headers,480                urllib_response,481            )482 483            if not request.stream_output:484                # Cause the raw stream to be exhausted immediately. We do it485                # this way instead of using preload_content because486                # preload_content will never buffer chunked responses487                http_response.content488 489            return http_response490        except URLLib3SSLError as e:491            raise SSLError(endpoint_url=request.url, error=e)492        except (NewConnectionError, socket.gaierror) as e:493            raise EndpointConnectionError(endpoint_url=request.url, error=e)494        except ProxyError as e:495            raise ProxyConnectionError(496                proxy_url=mask_proxy_url(proxy_url), error=e497            )498        except URLLib3ConnectTimeoutError as e:499            raise ConnectTimeoutError(endpoint_url=request.url, error=e)500        except URLLib3ReadTimeoutError as e:501            raise ReadTimeoutError(endpoint_url=request.url, error=e)502        except ProtocolError as e:503            raise ConnectionClosedError(504                error=e, request=request, endpoint_url=request.url505            )506        except Exception as e:507            message = 'Exception received when sending urllib3 HTTP request'508            logger.debug(message, exc_info=True)509            raise HTTPClientError(error=e)510