codekingpro/portable-devtools

1from __future__ import annotations2 3import errno4import logging5import queue6import sys7import typing8import warnings9import weakref10from socket import timeout as SocketTimeout11from types import TracebackType12 13from ._base_connection import _TYPE_BODY14from ._collections import HTTPHeaderDict15from ._request_methods import RequestMethods16from .connection import (17    BaseSSLError,18    BrokenPipeError,19    DummyConnection,20    HTTPConnection,21    HTTPException,22    HTTPSConnection,23    ProxyConfig,24    _wrap_proxy_error,25)26from .connection import port_by_scheme as port_by_scheme27from .exceptions import (28    ClosedPoolError,29    EmptyPoolError,30    FullPoolError,31    HostChangedError,32    InsecureRequestWarning,33    LocationValueError,34    MaxRetryError,35    NewConnectionError,36    ProtocolError,37    ProxyError,38    ReadTimeoutError,39    SSLError,40    TimeoutError,41)42from .response import BaseHTTPResponse43from .util.connection import is_connection_dropped44from .util.proxy import connection_requires_http_tunnel45from .util.request import _TYPE_BODY_POSITION, set_file_position46from .util.retry import Retry47from .util.ssl_match_hostname import CertificateError48from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_DEFAULT, Timeout49from .util.url import Url, _encode_target50from .util.url import _normalize_host as normalize_host51from .util.url import parse_url52from .util.util import to_str53 54if typing.TYPE_CHECKING:55    import ssl56 57    from typing_extensions import Self58 59    from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection60 61log = logging.getLogger(__name__)62 63_TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None]64 65 66# Pool objects67class ConnectionPool:68    """69    Base class for all connection pools, such as70    :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.71 72    .. note::73       ConnectionPool.urlopen() does not normalize or percent-encode target URIs74       which is useful if your target server doesn't support percent-encoded75       target URIs.76    """77 78    scheme: str | None = None79    QueueCls = queue.LifoQueue80 81    def __init__(self, host: str, port: int | None = None) -> None:82        if not host:83            raise LocationValueError("No host specified.")84 85        self.host = _normalize_host(host, scheme=self.scheme)86        self.port = port87 88        # This property uses 'normalize_host()' (not '_normalize_host()')89        # to avoid removing square braces around IPv6 addresses.90        # This value is sent to `HTTPConnection.set_tunnel()` if called91        # because square braces are required for HTTP CONNECT tunneling.92        self._tunnel_host = normalize_host(host, scheme=self.scheme).lower()93 94    def __str__(self) -> str:95        return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"96 97    def __enter__(self) -> Self:98        return self99 100    def __exit__(101        self,102        exc_type: type[BaseException] | None,103        exc_val: BaseException | None,104        exc_tb: TracebackType | None,105    ) -> typing.Literal[False]:106        self.close()107        # Return False to re-raise any potential exceptions108        return False109 110    def close(self) -> None:111        """112        Close all pooled connections and disable the pool.113        """114 115 116# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252117_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}118 119 120class HTTPConnectionPool(ConnectionPool, RequestMethods):121    """122    Thread-safe connection pool for one host.123 124    :param host:125        Host used for this HTTP Connection (e.g. "localhost"), passed into126        :class:`http.client.HTTPConnection`.127 128    :param port:129        Port used for this HTTP Connection (None is equivalent to 80), passed130        into :class:`http.client.HTTPConnection`.131 132    :param timeout:133        Socket timeout in seconds for each individual connection. This can134        be a float or integer, which sets the timeout for the HTTP request,135        or an instance of :class:`urllib3.util.Timeout` which gives you more136        fine-grained control over request timeouts. After the constructor has137        been parsed, this is always a `urllib3.util.Timeout` object.138 139    :param maxsize:140        Number of connections to save that can be reused. More than 1 is useful141        in multithreaded situations. If ``block`` is set to False, more142        connections will be created but they will not be saved once they've143        been used.144 145    :param block:146        If set to True, no more than ``maxsize`` connections will be used at147        a time. When no free connections are available, the call will block148        until a connection has been released. This is a useful side effect for149        particular multithreaded situations where one does not want to use more150        than maxsize connections per host to prevent flooding.151 152    :param headers:153        Headers to include with all requests, unless other headers are given154        explicitly.155 156    :param retries:157        Retry configuration to use by default with requests in this pool.158 159    :param _proxy:160        Parsed proxy URL, should not be used directly, instead, see161        :class:`urllib3.ProxyManager`162 163    :param _proxy_headers:164        A dictionary with proxy headers, should not be used directly,165        instead, see :class:`urllib3.ProxyManager`166 167    :param \\**conn_kw:168        Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,169        :class:`urllib3.connection.HTTPSConnection` instances.170    """171 172    scheme = "http"173    ConnectionCls: type[BaseHTTPConnection] | type[BaseHTTPSConnection] = HTTPConnection174 175    def __init__(176        self,177        host: str,178        port: int | None = None,179        timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,180        maxsize: int = 1,181        block: bool = False,182        headers: typing.Mapping[str, str] | None = None,183        retries: Retry | bool | int | None = None,184        _proxy: Url | None = None,185        _proxy_headers: typing.Mapping[str, str] | None = None,186        _proxy_config: ProxyConfig | None = None,187        **conn_kw: typing.Any,188    ):189        ConnectionPool.__init__(self, host, port)190        RequestMethods.__init__(self, headers)191 192        if not isinstance(timeout, Timeout):193            timeout = Timeout.from_float(timeout)194 195        if retries is None:196            retries = Retry.DEFAULT197 198        self.timeout = timeout199        self.retries = retries200 201        self.pool: queue.LifoQueue[typing.Any] | None = self.QueueCls(maxsize)202        self.block = block203 204        self.proxy = _proxy205        self.proxy_headers = _proxy_headers or {}206        self.proxy_config = _proxy_config207 208        # Fill the queue up so that doing get() on it will block properly209        for _ in range(maxsize):210            self.pool.put(None)211 212        # These are mostly for testing and debugging purposes.213        self.num_connections = 0214        self.num_requests = 0215        self.conn_kw = conn_kw216 217        if self.proxy:218            # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.219            # Defaulting `socket_options` to an empty list avoids it defaulting to220            # ``HTTPConnection.default_socket_options``.221            self.conn_kw.setdefault("socket_options", [])222 223            self.conn_kw["proxy"] = self.proxy224            self.conn_kw["proxy_config"] = self.proxy_config225 226        # Do not pass 'self' as callback to 'finalize'.227        # Then the 'finalize' would keep an endless living (leak) to self.228        # By just passing a reference to the pool allows the garbage collector229        # to free self if nobody else has a reference to it.230        pool = self.pool231 232        # Close all the HTTPConnections in the pool before the233        # HTTPConnectionPool object is garbage collected.234        weakref.finalize(self, _close_pool_connections, pool)235 236    def _new_conn(self) -> BaseHTTPConnection:237        """238        Return a fresh :class:`HTTPConnection`.239        """240        self.num_connections += 1241        log.debug(242            "Starting new HTTP connection (%d): %s:%s",243            self.num_connections,244            self.host,245            self.port or "80",246        )247 248        conn = self.ConnectionCls(249            host=self.host,250            port=self.port,251            timeout=self.timeout.connect_timeout,252            **self.conn_kw,253        )254        return conn255 256    def _get_conn(self, timeout: float | None = None) -> BaseHTTPConnection:257        """258        Get a connection. Will return a pooled connection if one is available.259 260        If no connections are available and :prop:`.block` is ``False``, then a261        fresh connection is returned.262 263        :param timeout:264            Seconds to wait before giving up and raising265            :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and266            :prop:`.block` is ``True``.267        """268        conn = None269 270        if self.pool is None:271            raise ClosedPoolError(self, "Pool is closed.")272 273        try:274            conn = self.pool.get(block=self.block, timeout=timeout)275 276        except AttributeError:  # self.pool is None277            raise ClosedPoolError(self, "Pool is closed.") from None  # Defensive:278 279        except queue.Empty:280            if self.block:281                raise EmptyPoolError(282                    self,283                    "Pool is empty and a new connection can't be opened due to blocking mode.",284                ) from None285            pass  # Oh well, we'll create a new connection then286 287        # If this is a persistent connection, check if it got disconnected288        if conn and is_connection_dropped(conn):289            log.debug("Resetting dropped connection: %s", self.host)290            conn.close()291 292        return conn or self._new_conn()293 294    def _put_conn(self, conn: BaseHTTPConnection | None) -> None:295        """296        Put a connection back into the pool.297 298        :param conn:299            Connection object for the current host and port as returned by300            :meth:`._new_conn` or :meth:`._get_conn`.301 302        If the pool is already full, the connection is closed and discarded303        because we exceeded maxsize. If connections are discarded frequently,304        then maxsize should be increased.305 306        If the pool is closed, then the connection will be closed and discarded.307        """308        if self.pool is not None:309            try:310                self.pool.put(conn, block=False)311                return  # Everything is dandy, done.312            except AttributeError:313                # self.pool is None.314                pass315            except queue.Full:316                # Connection never got put back into the pool, close it.317                if conn:318                    conn.close()319 320                if self.block:321                    # This should never happen if you got the conn from self._get_conn322                    raise FullPoolError(323                        self,324                        "Pool reached maximum size and no more connections are allowed.",325                    ) from None326 327                log.warning(328                    "Connection pool is full, discarding connection: %s. Connection pool size: %s",329                    self.host,330                    self.pool.qsize(),331                )332 333        # Connection never got put back into the pool, close it.334        if conn:335            conn.close()336 337    def _validate_conn(self, conn: BaseHTTPConnection) -> None:338        """339        Called right before a request is made, after the socket is created.340        """341 342    def _prepare_proxy(self, conn: BaseHTTPConnection) -> None:343        # Nothing to do for HTTP connections.344        pass345 346    def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout:347        """Helper that always returns a :class:`urllib3.util.Timeout`"""348        if timeout is _DEFAULT_TIMEOUT:349            return self.timeout.clone()350 351        if isinstance(timeout, Timeout):352            return timeout.clone()353        else:354            # User passed us an int/float. This is for backwards compatibility,355            # can be removed later356            return Timeout.from_float(timeout)357 358    def _raise_timeout(359        self,360        err: BaseSSLError | OSError | SocketTimeout,361        url: str,362        timeout_value: _TYPE_TIMEOUT | None,363    ) -> None:364        """Is the error actually a timeout? Will raise a ReadTimeout or pass"""365 366        if isinstance(err, SocketTimeout):367            raise ReadTimeoutError(368                self, url, f"Read timed out. (read timeout={timeout_value})"369            ) from err370 371        # See the above comment about EAGAIN in Python 3.372        if hasattr(err, "errno") and err.errno in _blocking_errnos:373            raise ReadTimeoutError(374                self, url, f"Read timed out. (read timeout={timeout_value})"375            ) from err376 377    def _make_request(378        self,379        conn: BaseHTTPConnection,380        method: str,381        url: str,382        body: _TYPE_BODY | None = None,383        headers: typing.Mapping[str, str] | None = None,384        retries: Retry | None = None,385        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,386        chunked: bool = False,387        response_conn: BaseHTTPConnection | None = None,388        preload_content: bool = True,389        decode_content: bool = True,390        enforce_content_length: bool = True,391    ) -> BaseHTTPResponse:392        """393        Perform a request on a given urllib connection object taken from our394        pool.395 396        :param conn:397            a connection from one of our connection pools398 399        :param method:400            HTTP request method (such as GET, POST, PUT, etc.)401 402        :param url:403            The URL to perform the request on.404 405        :param body:406            Data to send in the request body, either :class:`str`, :class:`bytes`,407            an iterable of :class:`str`/:class:`bytes`, or a file-like object.408 409        :param headers:410            Dictionary of custom headers to send, such as User-Agent,411            If-None-Match, etc. If None, pool headers are used. If provided,412            these headers completely replace any pool-specific headers.413 414        :param retries:415            Configure the number of retries to allow before raising a416            :class:`~urllib3.exceptions.MaxRetryError` exception.417 418            Pass ``None`` to retry until you receive a response. Pass a419            :class:`~urllib3.util.retry.Retry` object for fine-grained control420            over different types of retries.421            Pass an integer number to retry connection errors that many times,422            but no other types of errors. Pass zero to never retry.423 424            If ``False``, then retries are disabled and any exception is raised425            immediately. Also, instead of raising a MaxRetryError on redirects,426            the redirect response will be returned.427 428        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.429 430        :param timeout:431            If specified, overrides the default timeout for this one432            request. It may be a float (in seconds) or an instance of433            :class:`urllib3.util.Timeout`.434 435        :param chunked:436            If True, urllib3 will send the body using chunked transfer437            encoding. Otherwise, urllib3 will send the body using the standard438            content-length form. Defaults to False.439 440        :param response_conn:441            Set this to ``None`` if you will handle releasing the connection or442            set the connection to have the response release it.443 444        :param preload_content:445          If True, the response's body will be preloaded during construction.446 447        :param decode_content:448            If True, will attempt to decode the body based on the449            'content-encoding' header.450 451        :param enforce_content_length:452            Enforce content length checking. Body returned by server must match453            value of Content-Length header, if present. Otherwise, raise error.454        """455        self.num_requests += 1456 457        timeout_obj = self._get_timeout(timeout)458        timeout_obj.start_connect()459        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)460 461        try:462            # Trigger any extra validation we need to do.463            try:464                self._validate_conn(conn)465            except (SocketTimeout, BaseSSLError) as e:466                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)467                raise468 469        # _validate_conn() starts the connection to an HTTPS proxy470        # so we need to wrap errors with 'ProxyError' here too.471        except (472            OSError,473            NewConnectionError,474            TimeoutError,475            BaseSSLError,476            CertificateError,477            SSLError,478        ) as e:479            new_e: Exception = e480            if isinstance(e, (BaseSSLError, CertificateError)):481                new_e = SSLError(e)482            # If the connection didn't successfully connect to it's proxy483            # then there484            if isinstance(485                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)486            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):487                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)488            raise new_e489 490        # conn.request() calls http.client.*.request, not the method in491        # urllib3.request. It also calls makefile (recv) on the socket.492        try:493            conn.request(494                method,495                url,496                body=body,497                headers=headers,498                chunked=chunked,499                preload_content=preload_content,500                decode_content=decode_content,501                enforce_content_length=enforce_content_length,502            )503 504        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is505        # legitimately able to close the connection after sending a valid response.506        # With this behaviour, the received response is still readable.507        except BrokenPipeError:508            pass509        except OSError as e:510            # MacOS/Linux511            # EPROTOTYPE and ECONNRESET are needed on macOS512            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/513            # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.514            if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:515                raise516 517        # Reset the timeout for the recv() on the socket518        read_timeout = timeout_obj.read_timeout519 520        if not conn.is_closed:521            # In Python 3 socket.py will catch EAGAIN and return None when you522            # try and read into the file pointer created by http.client, which523            # instead raises a BadStatusLine exception. Instead of catching524            # the exception and assuming all BadStatusLine exceptions are read525            # timeouts, check for a zero timeout before making the request.526            if read_timeout == 0:527                raise ReadTimeoutError(528                    self, url, f"Read timed out. (read timeout={read_timeout})"529                )530            conn.timeout = read_timeout531 532        # Receive the response from the server533        try:534            response = conn.getresponse()535        except (BaseSSLError, OSError) as e:536            self._raise_timeout(err=e, url=url, timeout_value=read_timeout)537            raise538 539        # Set properties that are used by the pooling layer.540        response.retries = retries541        response._connection = response_conn  # type: ignore[attr-defined]542        response._pool = self  # type: ignore[attr-defined]543 544        log.debug(545            '%s://%s:%s "%s %s %s" %s %s',546            self.scheme,547            self.host,548            self.port,549            method,550            url,551            response.version_string,552            response.status,553            response.length_remaining,554        )555 556        return response557 558    def close(self) -> None:559        """560        Close all pooled connections and disable the pool.561        """562        if self.pool is None:563            return564        # Disable access to the pool565        old_pool, self.pool = self.pool, None566 567        # Close all the HTTPConnections in the pool.568        _close_pool_connections(old_pool)569 570    def is_same_host(self, url: str) -> bool:571        """572        Check if the given ``url`` is a member of the same host as this573        connection pool.574        """575        if url.startswith("/"):576            return True577 578        # TODO: Add optional support for socket.gethostbyname checking.579        scheme, _, host, port, *_ = parse_url(url)580        scheme = scheme or "http"581        if host is not None:582            host = _normalize_host(host, scheme=scheme)583 584        # Use explicit default port for comparison when none is given585        if self.port and not port:586            port = port_by_scheme.get(scheme)587        elif not self.port and port == port_by_scheme.get(scheme):588            port = None589 590        return (scheme, host, port) == (self.scheme, self.host, self.port)591 592    def urlopen(  # type: ignore[override]593        self,594        method: str,595        url: str,596        body: _TYPE_BODY | None = None,597        headers: typing.Mapping[str, str] | None = None,598        retries: Retry | bool | int | None = None,599        redirect: bool = True,600        assert_same_host: bool = True,601        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,602        pool_timeout: int | None = None,603        release_conn: bool | None = None,604        chunked: bool = False,605        body_pos: _TYPE_BODY_POSITION | None = None,606        preload_content: bool = True,607        decode_content: bool = True,608        **response_kw: typing.Any,609    ) -> BaseHTTPResponse:610        """611        Get a connection from the pool and perform an HTTP request. This is the612        lowest level call for making a request, so you'll need to specify all613        the raw details.614 615        .. note::616 617           More commonly, it's appropriate to use a convenience method618           such as :meth:`request`.619 620        .. note::621 622           `release_conn` will only behave as expected if623           `preload_content=False` because we want to make624           `preload_content=False` the default behaviour someday soon without625           breaking backwards compatibility.626 627        :param method:628            HTTP request method (such as GET, POST, PUT, etc.)629 630        :param url:631            The URL to perform the request on.632 633        :param body:634            Data to send in the request body, either :class:`str`, :class:`bytes`,635            an iterable of :class:`str`/:class:`bytes`, or a file-like object.636 637        :param headers:638            Dictionary of custom headers to send, such as User-Agent,639            If-None-Match, etc. If None, pool headers are used. If provided,640            these headers completely replace any pool-specific headers.641 642        :param retries:643            Configure the number of retries to allow before raising a644            :class:`~urllib3.exceptions.MaxRetryError` exception.645 646            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a647            :class:`~urllib3.util.retry.Retry` object for fine-grained control648            over different types of retries.649            Pass an integer number to retry connection errors that many times,650            but no other types of errors. Pass zero to never retry.651 652            If ``False``, then retries are disabled and any exception is raised653            immediately. Also, instead of raising a MaxRetryError on redirects,654            the redirect response will be returned.655 656        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.657 658        :param redirect:659            If True, automatically handle redirects (status codes 301, 302,660            303, 307, 308). Each redirect counts as a retry. Disabling retries661            will disable redirect, too.662 663        :param assert_same_host:664            If ``True``, will make sure that the host of the pool requests is665            consistent else will raise HostChangedError. When ``False``, you can666            use the pool on an HTTP proxy and request foreign hosts.667 668        :param timeout:669            If specified, overrides the default timeout for this one670            request. It may be a float (in seconds) or an instance of671            :class:`urllib3.util.Timeout`.672 673        :param pool_timeout:674            If set and the pool is set to block=True, then this method will675            block for ``pool_timeout`` seconds and raise EmptyPoolError if no676            connection is available within the time period.677 678        :param bool preload_content:679            If True, the response's body will be preloaded into memory.680 681        :param bool decode_content:682            If True, will attempt to decode the body based on the683            'content-encoding' header.684 685        :param release_conn:686            If False, then the urlopen call will not release the connection687            back into the pool once a response is received (but will release if688            you read the entire contents of the response such as when689            `preload_content=True`). This is useful if you're not preloading690            the response's content immediately. You will need to call691            ``r.release_conn()`` on the response ``r`` to return the connection692            back into the pool. If None, it takes the value of ``preload_content``693            which defaults to ``True``.694 695        :param bool chunked:696            If True, urllib3 will send the body using chunked transfer697            encoding. Otherwise, urllib3 will send the body using the standard698            content-length form. Defaults to False.699 700        :param int body_pos:701            Position to seek to in file-like body in the event of a retry or702            redirect. Typically this won't need to be set because urllib3 will703            auto-populate the value when needed.704        """705        # Ensure that the URL we're connecting to is properly encoded706        if url.startswith("/"):707            # URLs starting with / are inherently schemeless.708            url = to_str(_encode_target(url))709            destination_scheme = None710        else:711            parsed_url = parse_url(url)712            destination_scheme = parsed_url.scheme713            url = to_str(parsed_url.url)714 715        if headers is None:716            headers = self.headers717 718        if not isinstance(retries, Retry):719            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)720 721        if release_conn is None:722            release_conn = preload_content723 724        # Check host725        if assert_same_host and not self.is_same_host(url):726            raise HostChangedError(self, url, retries)727 728        conn = None729 730        # Track whether `conn` needs to be released before731        # returning/raising/recursing. Update this variable if necessary, and732        # leave `release_conn` constant throughout the function. That way, if733        # the function recurses, the original value of `release_conn` will be734        # passed down into the recursive call, and its value will be respected.735        #736        # See issue #651 [1] for details.737        #738        # [1] <https://github.com/urllib3/urllib3/issues/651>739        release_this_conn = release_conn740 741        http_tunnel_required = connection_requires_http_tunnel(742            self.proxy, self.proxy_config, destination_scheme743        )744 745        # Merge the proxy headers. Only done when not using HTTP CONNECT. We746        # have to copy the headers dict so we can safely change it without those747        # changes being reflected in anyone else's copy.748        if not http_tunnel_required:749            headers = headers.copy()  # type: ignore[attr-defined]750            headers.update(self.proxy_headers)  # type: ignore[union-attr]751 752        # Must keep the exception bound to a separate variable or else Python 3753        # complains about UnboundLocalError.754        err = None755 756        # Keep track of whether we cleanly exited the except block. This757        # ensures we do proper cleanup in finally.758        clean_exit = False759 760        # Rewind body position, if needed. Record current position761        # for future rewinds in the event of a redirect/retry.762        body_pos = set_file_position(body, body_pos)763 764        try:765            # Request a connection from the queue.766            timeout_obj = self._get_timeout(timeout)767            conn = self._get_conn(timeout=pool_timeout)768 769            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]770 771            # Is this a closed/new connection that requires CONNECT tunnelling?772            if self.proxy is not None and http_tunnel_required and conn.is_closed:773                try:774                    self._prepare_proxy(conn)775                except (BaseSSLError, OSError, SocketTimeout) as e:776                    self._raise_timeout(777                        err=e, url=self.proxy.url, timeout_value=conn.timeout778                    )779                    raise780 781            # If we're going to release the connection in ``finally:``, then782            # the response doesn't need to know about the connection. Otherwise783            # it will also try to release it and we'll have a double-release784            # mess.785            response_conn = conn if not release_conn else None786 787            # Make the request on the HTTPConnection object788            response = self._make_request(789                conn,790                method,791                url,792                timeout=timeout_obj,793                body=body,794                headers=headers,795                chunked=chunked,796                retries=retries,797                response_conn=response_conn,798                preload_content=preload_content,799                decode_content=decode_content,800                **response_kw,801            )802 803            # Everything went great!804            clean_exit = True805 806        except EmptyPoolError:807            # Didn't get a connection from the pool, no need to clean up808            clean_exit = True809            release_this_conn = False810            raise811 812        except (813            TimeoutError,814            HTTPException,815            OSError,816            ProtocolError,817            BaseSSLError,818            SSLError,819            CertificateError,820            ProxyError,821        ) as e:822            # Discard the connection for these exceptions. It will be823            # replaced during the next _get_conn() call.824            clean_exit = False825            new_e: Exception = e826            if isinstance(e, (BaseSSLError, CertificateError)):827                new_e = SSLError(e)828            if isinstance(829                new_e,830                (831                    OSError,832                    NewConnectionError,833                    TimeoutError,834                    SSLError,835                    HTTPException,836                ),837            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):838                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)839            elif isinstance(new_e, (OSError, HTTPException)):840                new_e = ProtocolError("Connection aborted.", new_e)841 842            retries = retries.increment(843                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]844            )845            retries.sleep()846 847            # Keep track of the error for the retry warning.848            err = e849 850        finally:851            if not clean_exit:852                # We hit some kind of exception, handled or otherwise. We need853                # to throw the connection away unless explicitly told not to.854                # Close the connection, set the variable to None, and make sure855                # we put the None back in the pool to avoid leaking it.856                if conn:857                    conn.close()858                    conn = None859                release_this_conn = True860 861            if release_this_conn:862                # Put the connection back to be reused. If the connection is863                # expired then it will be None, which will get replaced with a864                # fresh connection during _get_conn.865                self._put_conn(conn)866 867        if not conn:868            # Try again869            log.warning(870                "Retrying (%r) after connection broken by '%r': %s", retries, err, url871            )872            return self.urlopen(873                method,874                url,875                body,876                headers,877                retries,878                redirect,879                assert_same_host,880                timeout=timeout,881                pool_timeout=pool_timeout,882                release_conn=release_conn,883                chunked=chunked,884                body_pos=body_pos,885                preload_content=preload_content,886                decode_content=decode_content,887                **response_kw,888            )889 890        # Handle redirect?891        redirect_location = redirect and response.get_redirect_location()892        if redirect_location:893            if response.status == 303:894                # Change the method according to RFC 9110, Section 15.4.4.895                method = "GET"896                # And lose the body not to transfer anything sensitive.897                body = None898                headers = HTTPHeaderDict(headers)._prepare_for_method_change()899 900            # Strip headers marked as unsafe to forward to the redirected location.901            # Check remove_headers_on_redirect to avoid a potential network call within902            # self.is_same_host() which may use socket.gethostbyname() in the future.903            if retries.remove_headers_on_redirect and not self.is_same_host(904                redirect_location905            ):906                new_headers = headers.copy()  # type: ignore[union-attr]907                for header in headers:908                    if header.lower() in retries.remove_headers_on_redirect:909                        new_headers.pop(header, None)910                headers = new_headers911 912            try:913                retries = retries.increment(method, url, response=response, _pool=self)914            except MaxRetryError:915                if retries.raise_on_redirect:916                    response.drain_conn()917                    raise918                return response919 920            response.drain_conn()921            retries.sleep_for_retry(response)922            log.debug("Redirecting %s -> %s", url, redirect_location)923            return self.urlopen(924                method,925                redirect_location,926                body,927                headers,928                retries=retries,929                redirect=redirect,930                assert_same_host=assert_same_host,931                timeout=timeout,932                pool_timeout=pool_timeout,933                release_conn=release_conn,934                chunked=chunked,935                body_pos=body_pos,936                preload_content=preload_content,937                decode_content=decode_content,938                **response_kw,939            )940 941        # Check if we should retry the HTTP response.942        has_retry_after = bool(response.headers.get("Retry-After"))943        if retries.is_retry(method, response.status, has_retry_after):944            try:945                retries = retries.increment(method, url, response=response, _pool=self)946            except MaxRetryError:947                if retries.raise_on_status:948                    response.drain_conn()949                    raise950                return response951 952            response.drain_conn()953            retries.sleep(response)954            log.debug("Retry: %s", url)955            return self.urlopen(956                method,957                url,958                body,959                headers,960                retries=retries,961                redirect=redirect,962                assert_same_host=assert_same_host,963                timeout=timeout,964                pool_timeout=pool_timeout,965                release_conn=release_conn,966                chunked=chunked,967                body_pos=body_pos,968                preload_content=preload_content,969                decode_content=decode_content,970                **response_kw,971            )972 973        return response974 975 976class HTTPSConnectionPool(HTTPConnectionPool):977    """978    Same as :class:`.HTTPConnectionPool`, but HTTPS.979 980    :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,981    ``assert_hostname`` and ``host`` in this order to verify connections.982    If ``assert_hostname`` is False, no verification is done.983 984    The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,985    ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`986    is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade987    the connection socket into an SSL socket.988    """989 990    scheme = "https"991    ConnectionCls: type[BaseHTTPSConnection] = HTTPSConnection992 993    def __init__(994        self,995        host: str,996        port: int | None = None,997        timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,998        maxsize: int = 1,999        block: bool = False,1000        headers: typing.Mapping[str, str] | None = None,1001        retries: Retry | bool | int | None = None,1002        _proxy: Url | None = None,1003        _proxy_headers: typing.Mapping[str, str] | None = None,1004        key_file: str | None = None,1005        cert_file: str | None = None,1006        cert_reqs: int | str | None = None,1007        key_password: str | None = None,1008        ca_certs: str | None = None,1009        ssl_version: int | str | None = None,1010        ssl_minimum_version: ssl.TLSVersion | None = None,1011        ssl_maximum_version: ssl.TLSVersion | None = None,1012        assert_hostname: str | typing.Literal[False] | None = None,1013        assert_fingerprint: str | None = None,1014        ca_cert_dir: str | None = None,1015        **conn_kw: typing.Any,1016    ) -> None:1017        super().__init__(1018            host,1019            port,1020            timeout,1021            maxsize,1022            block,1023            headers,1024            retries,1025            _proxy,1026            _proxy_headers,1027            **conn_kw,1028        )1029 1030        self.key_file = key_file1031        self.cert_file = cert_file1032        self.cert_reqs = cert_reqs1033        self.key_password = key_password1034        self.ca_certs = ca_certs1035        self.ca_cert_dir = ca_cert_dir1036        self.ssl_version = ssl_version1037        self.ssl_minimum_version = ssl_minimum_version1038        self.ssl_maximum_version = ssl_maximum_version1039        self.assert_hostname = assert_hostname1040        self.assert_fingerprint = assert_fingerprint1041 1042    def _prepare_proxy(self, conn: HTTPSConnection) -> None:  # type: ignore[override]1043        """Establishes a tunnel connection through HTTP CONNECT."""1044        if self.proxy and self.proxy.scheme == "https":1045            tunnel_scheme = "https"1046        else:1047            tunnel_scheme = "http"1048 1049        conn.set_tunnel(1050            scheme=tunnel_scheme,1051            host=self._tunnel_host,1052            port=self.port,1053            headers=self.proxy_headers,1054        )1055        conn.connect()1056 1057    def _new_conn(self) -> BaseHTTPSConnection:1058        """1059        Return a fresh :class:`urllib3.connection.HTTPConnection`.1060        """1061        self.num_connections += 11062        log.debug(1063            "Starting new HTTPS connection (%d): %s:%s",1064            self.num_connections,1065            self.host,1066            self.port or "443",1067        )1068 1069        if not self.ConnectionCls or self.ConnectionCls is DummyConnection:  # type: ignore[comparison-overlap]1070            raise ImportError(1071                "Can't connect to HTTPS URL because the SSL module is not available."1072            )1073 1074        actual_host: str = self.host1075        actual_port = self.port1076        if self.proxy is not None and self.proxy.host is not None:1077            actual_host = self.proxy.host1078            actual_port = self.proxy.port1079 1080        return self.ConnectionCls(1081            host=actual_host,1082            port=actual_port,1083            timeout=self.timeout.connect_timeout,1084            cert_file=self.cert_file,1085            key_file=self.key_file,1086            key_password=self.key_password,1087            cert_reqs=self.cert_reqs,1088            ca_certs=self.ca_certs,1089            ca_cert_dir=self.ca_cert_dir,1090            assert_hostname=self.assert_hostname,1091            assert_fingerprint=self.assert_fingerprint,1092            ssl_version=self.ssl_version,1093            ssl_minimum_version=self.ssl_minimum_version,1094            ssl_maximum_version=self.ssl_maximum_version,1095            **self.conn_kw,1096        )1097 1098    def _validate_conn(self, conn: BaseHTTPConnection) -> None:1099        """1100        Called right before a request is made, after the socket is created.1101        """1102        super()._validate_conn(conn)1103 1104        # Force connect early to allow us to validate the connection.1105        if conn.is_closed:1106            conn.connect()1107 1108        # TODO revise this, see https://github.com/urllib3/urllib3/issues/27911109        if not conn.is_verified and not conn.proxy_is_verified:1110            warnings.warn(1111                (1112                    f"Unverified HTTPS request is being made to host '{conn.host}'. "1113                    "Adding certificate verification is strongly advised. See: "1114                    "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"1115                    "#tls-warnings"1116                ),1117                InsecureRequestWarning,1118            )1119 1120 1121def connection_from_url(url: str, **kw: typing.Any) -> HTTPConnectionPool:1122    """1123    Given a url, return an :class:`.ConnectionPool` instance of its host.1124 1125    This is a shortcut for not having to parse out the scheme, host, and port1126    of the url before creating an :class:`.ConnectionPool` instance.1127 1128    :param url:1129        Absolute URL string that must include the scheme. Port is optional.1130 1131    :param \\**kw:1132        Passes additional parameters to the constructor of the appropriate1133        :class:`.ConnectionPool`. Useful for specifying things like1134        timeout, maxsize, headers, etc.1135 1136    Example::1137 1138        >>> conn = connection_from_url('http://google.com/')1139        >>> r = conn.request('GET', '/')1140    """1141    scheme, _, host, port, *_ = parse_url(url)1142    scheme = scheme or "http"1143    port = port or port_by_scheme.get(scheme, 80)1144    if scheme == "https":1145        return HTTPSConnectionPool(host, port=port, **kw)  # type: ignore[arg-type]1146    else:1147        return HTTPConnectionPool(host, port=port, **kw)  # type: ignore[arg-type]1148 1149 1150@typing.overload1151def _normalize_host(host: None, scheme: str | None) -> None: ...1152 1153 1154@typing.overload1155def _normalize_host(host: str, scheme: str | None) -> str: ...1156 1157 1158def _normalize_host(host: str | None, scheme: str | None) -> str | None:1159    """1160    Normalize hosts for comparisons and use with sockets.1161    """1162 1163    host = normalize_host(host, scheme)1164 1165    # httplib doesn't like it when we include brackets in IPv6 addresses1166    # Specifically, if we include brackets but also pass the port then1167    # httplib crazily doubles up the square brackets on the Host header.1168    # Instead, we need to make sure we never pass ``None`` as the port.1169    # However, for backward compatibility reasons we can't actually1170    # *assert* that.  See http://bugs.python.org/issue285391171    if host and host.startswith("[") and host.endswith("]"):1172        host = host[1:-1]1173    return host1174 1175 1176def _url_from_pool(1177    pool: HTTPConnectionPool | HTTPSConnectionPool, path: str | None = None1178) -> str:1179    """Returns the URL from a given connection pool. This is mainly used for testing and logging."""1180    return Url(scheme=pool.scheme, host=pool.host, port=pool.port, path=path).url1181 1182 1183def _close_pool_connections(pool: queue.LifoQueue[typing.Any]) -> None:1184    """Drains a queue of connections and closes each one."""1185    try:1186        while True:1187            conn = pool.get(block=False)1188            if conn:1189                conn.close()1190    except queue.Empty:1191        pass  # Done.1192