codekingpro/portable-devtools

1from __future__ import absolute_import2 3import errno4import logging5import re6import socket7import sys8import warnings9from socket import error as SocketError10from socket import timeout as SocketTimeout11 12from ._collections import HTTPHeaderDict13from .connection import (14    BaseSSLError,15    BrokenPipeError,16    DummyConnection,17    HTTPConnection,18    HTTPException,19    HTTPSConnection,20    VerifiedHTTPSConnection,21    port_by_scheme,22)23from .exceptions import (24    ClosedPoolError,25    EmptyPoolError,26    HeaderParsingError,27    HostChangedError,28    InsecureRequestWarning,29    LocationValueError,30    MaxRetryError,31    NewConnectionError,32    ProtocolError,33    ProxyError,34    ReadTimeoutError,35    SSLError,36    TimeoutError,37)38from .packages import six39from .packages.six.moves import queue40from .request import RequestMethods41from .response import HTTPResponse42from .util.connection import is_connection_dropped43from .util.proxy import connection_requires_http_tunnel44from .util.queue import LifoQueue45from .util.request import set_file_position46from .util.response import assert_header_parsing47from .util.retry import Retry48from .util.ssl_match_hostname import CertificateError49from .util.timeout import Timeout50from .util.url import Url, _encode_target51from .util.url import _normalize_host as normalize_host52from .util.url import get_host, parse_url53 54try:  # Platform-specific: Python 355    import weakref56 57    weakref_finalize = weakref.finalize58except AttributeError:  # Platform-specific: Python 259    from .packages.backports.weakref_finalize import weakref_finalize60 61xrange = six.moves.xrange62 63log = logging.getLogger(__name__)64 65_Default = object()66 67 68# Pool objects69class ConnectionPool(object):70    """71    Base class for all connection pools, such as72    :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.73 74    .. note::75       ConnectionPool.urlopen() does not normalize or percent-encode target URIs76       which is useful if your target server doesn't support percent-encoded77       target URIs.78    """79 80    scheme = None81    QueueCls = LifoQueue82 83    def __init__(self, host, port=None):84        if not host:85            raise LocationValueError("No host specified.")86 87        self.host = _normalize_host(host, scheme=self.scheme)88        self._proxy_host = host.lower()89        self.port = port90 91    def __str__(self):92        return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port)93 94    def __enter__(self):95        return self96 97    def __exit__(self, exc_type, exc_val, exc_tb):98        self.close()99        # Return False to re-raise any potential exceptions100        return False101 102    def close(self):103        """104        Close all pooled connections and disable the pool.105        """106        pass107 108 109# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252110_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}111 112 113class HTTPConnectionPool(ConnectionPool, RequestMethods):114    """115    Thread-safe connection pool for one host.116 117    :param host:118        Host used for this HTTP Connection (e.g. "localhost"), passed into119        :class:`http.client.HTTPConnection`.120 121    :param port:122        Port used for this HTTP Connection (None is equivalent to 80), passed123        into :class:`http.client.HTTPConnection`.124 125    :param strict:126        Causes BadStatusLine to be raised if the status line can't be parsed127        as a valid HTTP/1.0 or 1.1 status line, passed into128        :class:`http.client.HTTPConnection`.129 130        .. note::131           Only works in Python 2. This parameter is ignored in Python 3.132 133    :param timeout:134        Socket timeout in seconds for each individual connection. This can135        be a float or integer, which sets the timeout for the HTTP request,136        or an instance of :class:`urllib3.util.Timeout` which gives you more137        fine-grained control over request timeouts. After the constructor has138        been parsed, this is always a `urllib3.util.Timeout` object.139 140    :param maxsize:141        Number of connections to save that can be reused. More than 1 is useful142        in multithreaded situations. If ``block`` is set to False, more143        connections will be created but they will not be saved once they've144        been used.145 146    :param block:147        If set to True, no more than ``maxsize`` connections will be used at148        a time. When no free connections are available, the call will block149        until a connection has been released. This is a useful side effect for150        particular multithreaded situations where one does not want to use more151        than maxsize connections per host to prevent flooding.152 153    :param headers:154        Headers to include with all requests, unless other headers are given155        explicitly.156 157    :param retries:158        Retry configuration to use by default with requests in this pool.159 160    :param _proxy:161        Parsed proxy URL, should not be used directly, instead, see162        :class:`urllib3.ProxyManager`163 164    :param _proxy_headers:165        A dictionary with proxy headers, should not be used directly,166        instead, see :class:`urllib3.ProxyManager`167 168    :param \\**conn_kw:169        Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,170        :class:`urllib3.connection.HTTPSConnection` instances.171    """172 173    scheme = "http"174    ConnectionCls = HTTPConnection175    ResponseCls = HTTPResponse176 177    def __init__(178        self,179        host,180        port=None,181        strict=False,182        timeout=Timeout.DEFAULT_TIMEOUT,183        maxsize=1,184        block=False,185        headers=None,186        retries=None,187        _proxy=None,188        _proxy_headers=None,189        _proxy_config=None,190        **conn_kw191    ):192        ConnectionPool.__init__(self, host, port)193        RequestMethods.__init__(self, headers)194 195        self.strict = strict196 197        if not isinstance(timeout, Timeout):198            timeout = Timeout.from_float(timeout)199 200        if retries is None:201            retries = Retry.DEFAULT202 203        self.timeout = timeout204        self.retries = retries205 206        self.pool = self.QueueCls(maxsize)207        self.block = block208 209        self.proxy = _proxy210        self.proxy_headers = _proxy_headers or {}211        self.proxy_config = _proxy_config212 213        # Fill the queue up so that doing get() on it will block properly214        for _ in xrange(maxsize):215            self.pool.put(None)216 217        # These are mostly for testing and debugging purposes.218        self.num_connections = 0219        self.num_requests = 0220        self.conn_kw = conn_kw221 222        if self.proxy:223            # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.224            # We cannot know if the user has added default socket options, so we cannot replace the225            # list.226            self.conn_kw.setdefault("socket_options", [])227 228            self.conn_kw["proxy"] = self.proxy229            self.conn_kw["proxy_config"] = self.proxy_config230 231        # Do not pass 'self' as callback to 'finalize'.232        # Then the 'finalize' would keep an endless living (leak) to self.233        # By just passing a reference to the pool allows the garbage collector234        # to free self if nobody else has a reference to it.235        pool = self.pool236 237        # Close all the HTTPConnections in the pool before the238        # HTTPConnectionPool object is garbage collected.239        weakref_finalize(self, _close_pool_connections, pool)240 241    def _new_conn(self):242        """243        Return a fresh :class:`HTTPConnection`.244        """245        self.num_connections += 1246        log.debug(247            "Starting new HTTP connection (%d): %s:%s",248            self.num_connections,249            self.host,250            self.port or "80",251        )252 253        conn = self.ConnectionCls(254            host=self.host,255            port=self.port,256            timeout=self.timeout.connect_timeout,257            strict=self.strict,258            **self.conn_kw259        )260        return conn261 262    def _get_conn(self, timeout=None):263        """264        Get a connection. Will return a pooled connection if one is available.265 266        If no connections are available and :prop:`.block` is ``False``, then a267        fresh connection is returned.268 269        :param timeout:270            Seconds to wait before giving up and raising271            :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and272            :prop:`.block` is ``True``.273        """274        conn = None275        try:276            conn = self.pool.get(block=self.block, timeout=timeout)277 278        except AttributeError:  # self.pool is None279            raise ClosedPoolError(self, "Pool is closed.")280 281        except queue.Empty:282            if self.block:283                raise EmptyPoolError(284                    self,285                    "Pool reached maximum size and no more connections are allowed.",286                )287            pass  # Oh well, we'll create a new connection then288 289        # If this is a persistent connection, check if it got disconnected290        if conn and is_connection_dropped(conn):291            log.debug("Resetting dropped connection: %s", self.host)292            conn.close()293            if getattr(conn, "auto_open", 1) == 0:294                # This is a proxied connection that has been mutated by295                # http.client._tunnel() and cannot be reused (since it would296                # attempt to bypass the proxy)297                conn = None298 299        return conn or self._new_conn()300 301    def _put_conn(self, conn):302        """303        Put a connection back into the pool.304 305        :param conn:306            Connection object for the current host and port as returned by307            :meth:`._new_conn` or :meth:`._get_conn`.308 309        If the pool is already full, the connection is closed and discarded310        because we exceeded maxsize. If connections are discarded frequently,311        then maxsize should be increased.312 313        If the pool is closed, then the connection will be closed and discarded.314        """315        try:316            self.pool.put(conn, block=False)317            return  # Everything is dandy, done.318        except AttributeError:319            # self.pool is None.320            pass321        except queue.Full:322            # This should never happen if self.block == True323            log.warning(324                "Connection pool is full, discarding connection: %s. Connection pool size: %s",325                self.host,326                self.pool.qsize(),327            )328        # Connection never got put back into the pool, close it.329        if conn:330            conn.close()331 332    def _validate_conn(self, conn):333        """334        Called right before a request is made, after the socket is created.335        """336        pass337 338    def _prepare_proxy(self, conn):339        # Nothing to do for HTTP connections.340        pass341 342    def _get_timeout(self, timeout):343        """Helper that always returns a :class:`urllib3.util.Timeout`"""344        if timeout is _Default:345            return self.timeout.clone()346 347        if isinstance(timeout, Timeout):348            return timeout.clone()349        else:350            # User passed us an int/float. This is for backwards compatibility,351            # can be removed later352            return Timeout.from_float(timeout)353 354    def _raise_timeout(self, err, url, timeout_value):355        """Is the error actually a timeout? Will raise a ReadTimeout or pass"""356 357        if isinstance(err, SocketTimeout):358            raise ReadTimeoutError(359                self, url, "Read timed out. (read timeout=%s)" % timeout_value360            )361 362        # See the above comment about EAGAIN in Python 3. In Python 2 we have363        # to specifically catch it and throw the timeout error364        if hasattr(err, "errno") and err.errno in _blocking_errnos:365            raise ReadTimeoutError(366                self, url, "Read timed out. (read timeout=%s)" % timeout_value367            )368 369        # Catch possible read timeouts thrown as SSL errors. If not the370        # case, rethrow the original. We need to do this because of:371        # http://bugs.python.org/issue10272372        if "timed out" in str(err) or "did not complete (read)" in str(373            err374        ):  # Python < 2.7.4375            raise ReadTimeoutError(376                self, url, "Read timed out. (read timeout=%s)" % timeout_value377            )378 379    def _make_request(380        self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw381    ):382        """383        Perform a request on a given urllib connection object taken from our384        pool.385 386        :param conn:387            a connection from one of our connection pools388 389        :param timeout:390            Socket timeout in seconds for the request. This can be a391            float or integer, which will set the same timeout value for392            the socket connect and the socket read, or an instance of393            :class:`urllib3.util.Timeout`, which gives you more fine-grained394            control over your timeouts.395        """396        self.num_requests += 1397 398        timeout_obj = self._get_timeout(timeout)399        timeout_obj.start_connect()400        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)401 402        # Trigger any extra validation we need to do.403        try:404            self._validate_conn(conn)405        except (SocketTimeout, BaseSSLError) as e:406            # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.407            self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)408            raise409 410        # conn.request() calls http.client.*.request, not the method in411        # urllib3.request. It also calls makefile (recv) on the socket.412        try:413            if chunked:414                conn.request_chunked(method, url, **httplib_request_kw)415            else:416                conn.request(method, url, **httplib_request_kw)417 418        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is419        # legitimately able to close the connection after sending a valid response.420        # With this behaviour, the received response is still readable.421        except BrokenPipeError:422            # Python 3423            pass424        except IOError as e:425            # Python 2 and macOS/Linux426            # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS427            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/428            if e.errno not in {429                errno.EPIPE,430                errno.ESHUTDOWN,431                errno.EPROTOTYPE,432            }:433                raise434 435        # Reset the timeout for the recv() on the socket436        read_timeout = timeout_obj.read_timeout437 438        # App Engine doesn't have a sock attr439        if getattr(conn, "sock", None):440            # In Python 3 socket.py will catch EAGAIN and return None when you441            # try and read into the file pointer created by http.client, which442            # instead raises a BadStatusLine exception. Instead of catching443            # the exception and assuming all BadStatusLine exceptions are read444            # timeouts, check for a zero timeout before making the request.445            if read_timeout == 0:446                raise ReadTimeoutError(447                    self, url, "Read timed out. (read timeout=%s)" % read_timeout448                )449            if read_timeout is Timeout.DEFAULT_TIMEOUT:450                conn.sock.settimeout(socket.getdefaulttimeout())451            else:  # None or a value452                conn.sock.settimeout(read_timeout)453 454        # Receive the response from the server455        try:456            try:457                # Python 2.7, use buffering of HTTP responses458                httplib_response = conn.getresponse(buffering=True)459            except TypeError:460                # Python 3461                try:462                    httplib_response = conn.getresponse()463                except BaseException as e:464                    # Remove the TypeError from the exception chain in465                    # Python 3 (including for exceptions like SystemExit).466                    # Otherwise it looks like a bug in the code.467                    six.raise_from(e, None)468        except (SocketTimeout, BaseSSLError, SocketError) as e:469            self._raise_timeout(err=e, url=url, timeout_value=read_timeout)470            raise471 472        # AppEngine doesn't have a version attr.473        http_version = getattr(conn, "_http_vsn_str", "HTTP/?")474        log.debug(475            '%s://%s:%s "%s %s %s" %s %s',476            self.scheme,477            self.host,478            self.port,479            method,480            url,481            http_version,482            httplib_response.status,483            httplib_response.length,484        )485 486        try:487            assert_header_parsing(httplib_response.msg)488        except (HeaderParsingError, TypeError) as hpe:  # Platform-specific: Python 3489            log.warning(490                "Failed to parse headers (url=%s): %s",491                self._absolute_url(url),492                hpe,493                exc_info=True,494            )495 496        return httplib_response497 498    def _absolute_url(self, path):499        return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url500 501    def close(self):502        """503        Close all pooled connections and disable the pool.504        """505        if self.pool is None:506            return507        # Disable access to the pool508        old_pool, self.pool = self.pool, None509 510        # Close all the HTTPConnections in the pool.511        _close_pool_connections(old_pool)512 513    def is_same_host(self, url):514        """515        Check if the given ``url`` is a member of the same host as this516        connection pool.517        """518        if url.startswith("/"):519            return True520 521        # TODO: Add optional support for socket.gethostbyname checking.522        scheme, host, port = get_host(url)523        if host is not None:524            host = _normalize_host(host, scheme=scheme)525 526        # Use explicit default port for comparison when none is given527        if self.port and not port:528            port = port_by_scheme.get(scheme)529        elif not self.port and port == port_by_scheme.get(scheme):530            port = None531 532        return (scheme, host, port) == (self.scheme, self.host, self.port)533 534    def urlopen(535        self,536        method,537        url,538        body=None,539        headers=None,540        retries=None,541        redirect=True,542        assert_same_host=True,543        timeout=_Default,544        pool_timeout=None,545        release_conn=None,546        chunked=False,547        body_pos=None,548        **response_kw549    ):550        """551        Get a connection from the pool and perform an HTTP request. This is the552        lowest level call for making a request, so you'll need to specify all553        the raw details.554 555        .. note::556 557           More commonly, it's appropriate to use a convenience method provided558           by :class:`.RequestMethods`, such as :meth:`request`.559 560        .. note::561 562           `release_conn` will only behave as expected if563           `preload_content=False` because we want to make564           `preload_content=False` the default behaviour someday soon without565           breaking backwards compatibility.566 567        :param method:568            HTTP request method (such as GET, POST, PUT, etc.)569 570        :param url:571            The URL to perform the request on.572 573        :param body:574            Data to send in the request body, either :class:`str`, :class:`bytes`,575            an iterable of :class:`str`/:class:`bytes`, or a file-like object.576 577        :param headers:578            Dictionary of custom headers to send, such as User-Agent,579            If-None-Match, etc. If None, pool headers are used. If provided,580            these headers completely replace any pool-specific headers.581 582        :param retries:583            Configure the number of retries to allow before raising a584            :class:`~urllib3.exceptions.MaxRetryError` exception.585 586            Pass ``None`` to retry until you receive a response. Pass a587            :class:`~urllib3.util.retry.Retry` object for fine-grained control588            over different types of retries.589            Pass an integer number to retry connection errors that many times,590            but no other types of errors. Pass zero to never retry.591 592            If ``False``, then retries are disabled and any exception is raised593            immediately. Also, instead of raising a MaxRetryError on redirects,594            the redirect response will be returned.595 596        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.597 598        :param redirect:599            If True, automatically handle redirects (status codes 301, 302,600            303, 307, 308). Each redirect counts as a retry. Disabling retries601            will disable redirect, too.602 603        :param assert_same_host:604            If ``True``, will make sure that the host of the pool requests is605            consistent else will raise HostChangedError. When ``False``, you can606            use the pool on an HTTP proxy and request foreign hosts.607 608        :param timeout:609            If specified, overrides the default timeout for this one610            request. It may be a float (in seconds) or an instance of611            :class:`urllib3.util.Timeout`.612 613        :param pool_timeout:614            If set and the pool is set to block=True, then this method will615            block for ``pool_timeout`` seconds and raise EmptyPoolError if no616            connection is available within the time period.617 618        :param release_conn:619            If False, then the urlopen call will not release the connection620            back into the pool once a response is received (but will release if621            you read the entire contents of the response such as when622            `preload_content=True`). This is useful if you're not preloading623            the response's content immediately. You will need to call624            ``r.release_conn()`` on the response ``r`` to return the connection625            back into the pool. If None, it takes the value of626            ``response_kw.get('preload_content', True)``.627 628        :param chunked:629            If True, urllib3 will send the body using chunked transfer630            encoding. Otherwise, urllib3 will send the body using the standard631            content-length form. Defaults to False.632 633        :param int body_pos:634            Position to seek to in file-like body in the event of a retry or635            redirect. Typically this won't need to be set because urllib3 will636            auto-populate the value when needed.637 638        :param \\**response_kw:639            Additional parameters are passed to640            :meth:`urllib3.response.HTTPResponse.from_httplib`641        """642 643        parsed_url = parse_url(url)644        destination_scheme = parsed_url.scheme645 646        if headers is None:647            headers = self.headers648 649        if not isinstance(retries, Retry):650            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)651 652        if release_conn is None:653            release_conn = response_kw.get("preload_content", True)654 655        # Check host656        if assert_same_host and not self.is_same_host(url):657            raise HostChangedError(self, url, retries)658 659        # Ensure that the URL we're connecting to is properly encoded660        if url.startswith("/"):661            url = six.ensure_str(_encode_target(url))662        else:663            url = six.ensure_str(parsed_url.url)664 665        conn = None666 667        # Track whether `conn` needs to be released before668        # returning/raising/recursing. Update this variable if necessary, and669        # leave `release_conn` constant throughout the function. That way, if670        # the function recurses, the original value of `release_conn` will be671        # passed down into the recursive call, and its value will be respected.672        #673        # See issue #651 [1] for details.674        #675        # [1] <https://github.com/urllib3/urllib3/issues/651>676        release_this_conn = release_conn677 678        http_tunnel_required = connection_requires_http_tunnel(679            self.proxy, self.proxy_config, destination_scheme680        )681 682        # Merge the proxy headers. Only done when not using HTTP CONNECT. We683        # have to copy the headers dict so we can safely change it without those684        # changes being reflected in anyone else's copy.685        if not http_tunnel_required:686            headers = headers.copy()687            headers.update(self.proxy_headers)688 689        # Must keep the exception bound to a separate variable or else Python 3690        # complains about UnboundLocalError.691        err = None692 693        # Keep track of whether we cleanly exited the except block. This694        # ensures we do proper cleanup in finally.695        clean_exit = False696 697        # Rewind body position, if needed. Record current position698        # for future rewinds in the event of a redirect/retry.699        body_pos = set_file_position(body, body_pos)700 701        try:702            # Request a connection from the queue.703            timeout_obj = self._get_timeout(timeout)704            conn = self._get_conn(timeout=pool_timeout)705 706            conn.timeout = timeout_obj.connect_timeout707 708            is_new_proxy_conn = self.proxy is not None and not getattr(709                conn, "sock", None710            )711            if is_new_proxy_conn and http_tunnel_required:712                self._prepare_proxy(conn)713 714            # Make the request on the httplib connection object.715            httplib_response = self._make_request(716                conn,717                method,718                url,719                timeout=timeout_obj,720                body=body,721                headers=headers,722                chunked=chunked,723            )724 725            # If we're going to release the connection in ``finally:``, then726            # the response doesn't need to know about the connection. Otherwise727            # it will also try to release it and we'll have a double-release728            # mess.729            response_conn = conn if not release_conn else None730 731            # Pass method to Response for length checking732            response_kw["request_method"] = method733 734            # Import httplib's response into our own wrapper object735            response = self.ResponseCls.from_httplib(736                httplib_response,737                pool=self,738                connection=response_conn,739                retries=retries,740                **response_kw741            )742 743            # Everything went great!744            clean_exit = True745 746        except EmptyPoolError:747            # Didn't get a connection from the pool, no need to clean up748            clean_exit = True749            release_this_conn = False750            raise751 752        except (753            TimeoutError,754            HTTPException,755            SocketError,756            ProtocolError,757            BaseSSLError,758            SSLError,759            CertificateError,760        ) as e:761            # Discard the connection for these exceptions. It will be762            # replaced during the next _get_conn() call.763            clean_exit = False764 765            def _is_ssl_error_message_from_http_proxy(ssl_error):766                # We're trying to detect the message 'WRONG_VERSION_NUMBER' but767                # SSLErrors are kinda all over the place when it comes to the message,768                # so we try to cover our bases here!769                message = " ".join(re.split("[^a-z]", str(ssl_error).lower()))770                return (771                    "wrong version number" in message or "unknown protocol" in message772                )773 774            # Try to detect a common user error with proxies which is to775            # set an HTTP proxy to be HTTPS when it should be 'http://'776            # (ie {'http': 'http://proxy', 'https': 'https://proxy'})777            # Instead we add a nice error message and point to a URL.778            if (779                isinstance(e, BaseSSLError)780                and self.proxy781                and _is_ssl_error_message_from_http_proxy(e)782                and conn.proxy783                and conn.proxy.scheme == "https"784            ):785                e = ProxyError(786                    "Your proxy appears to only use HTTP and not HTTPS, "787                    "try changing your proxy URL to be HTTP. See: "788                    "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html"789                    "#https-proxy-error-http-proxy",790                    SSLError(e),791                )792            elif isinstance(e, (BaseSSLError, CertificateError)):793                e = SSLError(e)794            elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy:795                e = ProxyError("Cannot connect to proxy.", e)796            elif isinstance(e, (SocketError, HTTPException)):797                e = ProtocolError("Connection aborted.", e)798 799            retries = retries.increment(800                method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]801            )802            retries.sleep()803 804            # Keep track of the error for the retry warning.805            err = e806 807        finally:808            if not clean_exit:809                # We hit some kind of exception, handled or otherwise. We need810                # to throw the connection away unless explicitly told not to.811                # Close the connection, set the variable to None, and make sure812                # we put the None back in the pool to avoid leaking it.813                conn = conn and conn.close()814                release_this_conn = True815 816            if release_this_conn:817                # Put the connection back to be reused. If the connection is818                # expired then it will be None, which will get replaced with a819                # fresh connection during _get_conn.820                self._put_conn(conn)821 822        if not conn:823            # Try again824            log.warning(825                "Retrying (%r) after connection broken by '%r': %s", retries, err, url826            )827            return self.urlopen(828                method,829                url,830                body,831                headers,832                retries,833                redirect,834                assert_same_host,835                timeout=timeout,836                pool_timeout=pool_timeout,837                release_conn=release_conn,838                chunked=chunked,839                body_pos=body_pos,840                **response_kw841            )842 843        # Handle redirect?844        redirect_location = redirect and response.get_redirect_location()845        if redirect_location:846            if response.status == 303:847                # Change the method according to RFC 9110, Section 15.4.4.848                method = "GET"849                # And lose the body not to transfer anything sensitive.850                body = None851                headers = HTTPHeaderDict(headers)._prepare_for_method_change()852 853            try:854                retries = retries.increment(method, url, response=response, _pool=self)855            except MaxRetryError:856                if retries.raise_on_redirect:857                    response.drain_conn()858                    raise859                return response860 861            response.drain_conn()862            retries.sleep_for_retry(response)863            log.debug("Redirecting %s -> %s", url, redirect_location)864            return self.urlopen(865                method,866                redirect_location,867                body,868                headers,869                retries=retries,870                redirect=redirect,871                assert_same_host=assert_same_host,872                timeout=timeout,873                pool_timeout=pool_timeout,874                release_conn=release_conn,875                chunked=chunked,876                body_pos=body_pos,877                **response_kw878            )879 880        # Check if we should retry the HTTP response.881        has_retry_after = bool(response.headers.get("Retry-After"))882        if retries.is_retry(method, response.status, has_retry_after):883            try:884                retries = retries.increment(method, url, response=response, _pool=self)885            except MaxRetryError:886                if retries.raise_on_status:887                    response.drain_conn()888                    raise889                return response890 891            response.drain_conn()892            retries.sleep(response)893            log.debug("Retry: %s", url)894            return self.urlopen(895                method,896                url,897                body,898                headers,899                retries=retries,900                redirect=redirect,901                assert_same_host=assert_same_host,902                timeout=timeout,903                pool_timeout=pool_timeout,904                release_conn=release_conn,905                chunked=chunked,906                body_pos=body_pos,907                **response_kw908            )909 910        return response911 912 913class HTTPSConnectionPool(HTTPConnectionPool):914    """915    Same as :class:`.HTTPConnectionPool`, but HTTPS.916 917    :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,918    ``assert_hostname`` and ``host`` in this order to verify connections.919    If ``assert_hostname`` is False, no verification is done.920 921    The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,922    ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`923    is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade924    the connection socket into an SSL socket.925    """926 927    scheme = "https"928    ConnectionCls = HTTPSConnection929 930    def __init__(931        self,932        host,933        port=None,934        strict=False,935        timeout=Timeout.DEFAULT_TIMEOUT,936        maxsize=1,937        block=False,938        headers=None,939        retries=None,940        _proxy=None,941        _proxy_headers=None,942        key_file=None,943        cert_file=None,944        cert_reqs=None,945        key_password=None,946        ca_certs=None,947        ssl_version=None,948        assert_hostname=None,949        assert_fingerprint=None,950        ca_cert_dir=None,951        **conn_kw952    ):953 954        HTTPConnectionPool.__init__(955            self,956            host,957            port,958            strict,959            timeout,960            maxsize,961            block,962            headers,963            retries,964            _proxy,965            _proxy_headers,966            **conn_kw967        )968 969        self.key_file = key_file970        self.cert_file = cert_file971        self.cert_reqs = cert_reqs972        self.key_password = key_password973        self.ca_certs = ca_certs974        self.ca_cert_dir = ca_cert_dir975        self.ssl_version = ssl_version976        self.assert_hostname = assert_hostname977        self.assert_fingerprint = assert_fingerprint978 979    def _prepare_conn(self, conn):980        """981        Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`982        and establish the tunnel if proxy is used.983        """984 985        if isinstance(conn, VerifiedHTTPSConnection):986            conn.set_cert(987                key_file=self.key_file,988                key_password=self.key_password,989                cert_file=self.cert_file,990                cert_reqs=self.cert_reqs,991                ca_certs=self.ca_certs,992                ca_cert_dir=self.ca_cert_dir,993                assert_hostname=self.assert_hostname,994                assert_fingerprint=self.assert_fingerprint,995            )996            conn.ssl_version = self.ssl_version997        return conn998 999    def _prepare_proxy(self, conn):1000        """1001        Establishes a tunnel connection through HTTP CONNECT.1002 1003        Tunnel connection is established early because otherwise httplib would1004        improperly set Host: header to proxy's IP:port.1005        """1006 1007        conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)1008 1009        if self.proxy.scheme == "https":1010            conn.tls_in_tls_required = True1011 1012        conn.connect()1013 1014    def _new_conn(self):1015        """1016        Return a fresh :class:`http.client.HTTPSConnection`.1017        """1018        self.num_connections += 11019        log.debug(1020            "Starting new HTTPS connection (%d): %s:%s",1021            self.num_connections,1022            self.host,1023            self.port or "443",1024        )1025 1026        if not self.ConnectionCls or self.ConnectionCls is DummyConnection:1027            raise SSLError(1028                "Can't connect to HTTPS URL because the SSL module is not available."1029            )1030 1031        actual_host = self.host1032        actual_port = self.port1033        if self.proxy is not None:1034            actual_host = self.proxy.host1035            actual_port = self.proxy.port1036 1037        conn = self.ConnectionCls(1038            host=actual_host,1039            port=actual_port,1040            timeout=self.timeout.connect_timeout,1041            strict=self.strict,1042            cert_file=self.cert_file,1043            key_file=self.key_file,1044            key_password=self.key_password,1045            **self.conn_kw1046        )1047 1048        return self._prepare_conn(conn)1049 1050    def _validate_conn(self, conn):1051        """1052        Called right before a request is made, after the socket is created.1053        """1054        super(HTTPSConnectionPool, self)._validate_conn(conn)1055 1056        # Force connect early to allow us to validate the connection.1057        if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`1058            conn.connect()1059 1060        if not conn.is_verified:1061            warnings.warn(1062                (1063                    "Unverified HTTPS request is being made to host '%s'. "1064                    "Adding certificate verification is strongly advised. See: "1065                    "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html"1066                    "#ssl-warnings" % conn.host1067                ),1068                InsecureRequestWarning,1069            )1070 1071        if getattr(conn, "proxy_is_verified", None) is False:1072            warnings.warn(1073                (1074                    "Unverified HTTPS connection done to an HTTPS proxy. "1075                    "Adding certificate verification is strongly advised. See: "1076                    "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html"1077                    "#ssl-warnings"1078                ),1079                InsecureRequestWarning,1080            )1081 1082 1083def connection_from_url(url, **kw):1084    """1085    Given a url, return an :class:`.ConnectionPool` instance of its host.1086 1087    This is a shortcut for not having to parse out the scheme, host, and port1088    of the url before creating an :class:`.ConnectionPool` instance.1089 1090    :param url:1091        Absolute URL string that must include the scheme. Port is optional.1092 1093    :param \\**kw:1094        Passes additional parameters to the constructor of the appropriate1095        :class:`.ConnectionPool`. Useful for specifying things like1096        timeout, maxsize, headers, etc.1097 1098    Example::1099 1100        >>> conn = connection_from_url('http://google.com/')1101        >>> r = conn.request('GET', '/')1102    """1103    scheme, host, port = get_host(url)1104    port = port or port_by_scheme.get(scheme, 80)1105    if scheme == "https":1106        return HTTPSConnectionPool(host, port=port, **kw)1107    else:1108        return HTTPConnectionPool(host, port=port, **kw)1109 1110 1111def _normalize_host(host, scheme):1112    """1113    Normalize hosts for comparisons and use with sockets.1114    """1115 1116    host = normalize_host(host, scheme)1117 1118    # httplib doesn't like it when we include brackets in IPv6 addresses1119    # Specifically, if we include brackets but also pass the port then1120    # httplib crazily doubles up the square brackets on the Host header.1121    # Instead, we need to make sure we never pass ``None`` as the port.1122    # However, for backward compatibility reasons we can't actually1123    # *assert* that.  See http://bugs.python.org/issue285391124    if host.startswith("[") and host.endswith("]"):1125        host = host[1:-1]1126    return host1127 1128 1129def _close_pool_connections(pool):1130    """Drains a queue of connections and closes each one."""1131    try:1132        while True:1133            conn = pool.get(block=False)1134            if conn:1135                conn.close()1136    except queue.Empty:1137        pass  # Done.1138