codekingpro/portable-devtools

1from __future__ import absolute_import2 3import io4import logging5import sys6import warnings7import zlib8from contextlib import contextmanager9from socket import error as SocketError10from socket import timeout as SocketTimeout11 12try:13    try:14        import brotlicffi as brotli15    except ImportError:16        import brotli17except ImportError:18    brotli = None19 20from . import util21from ._collections import HTTPHeaderDict22from .connection import BaseSSLError, HTTPException23from .exceptions import (24    BodyNotHttplibCompatible,25    DecodeError,26    HTTPError,27    IncompleteRead,28    InvalidChunkLength,29    InvalidHeader,30    ProtocolError,31    ReadTimeoutError,32    ResponseNotChunked,33    SSLError,34)35from .packages import six36from .util.response import is_fp_closed, is_response_to_head37 38log = logging.getLogger(__name__)39 40 41class DeflateDecoder(object):42    def __init__(self):43        self._first_try = True44        self._data = b""45        self._obj = zlib.decompressobj()46 47    def __getattr__(self, name):48        return getattr(self._obj, name)49 50    def decompress(self, data):51        if not data:52            return data53 54        if not self._first_try:55            return self._obj.decompress(data)56 57        self._data += data58        try:59            decompressed = self._obj.decompress(data)60            if decompressed:61                self._first_try = False62                self._data = None63            return decompressed64        except zlib.error:65            self._first_try = False66            self._obj = zlib.decompressobj(-zlib.MAX_WBITS)67            try:68                return self.decompress(self._data)69            finally:70                self._data = None71 72 73class GzipDecoderState(object):74 75    FIRST_MEMBER = 076    OTHER_MEMBERS = 177    SWALLOW_DATA = 278 79 80class GzipDecoder(object):81    def __init__(self):82        self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)83        self._state = GzipDecoderState.FIRST_MEMBER84 85    def __getattr__(self, name):86        return getattr(self._obj, name)87 88    def decompress(self, data):89        ret = bytearray()90        if self._state == GzipDecoderState.SWALLOW_DATA or not data:91            return bytes(ret)92        while True:93            try:94                ret += self._obj.decompress(data)95            except zlib.error:96                previous_state = self._state97                # Ignore data after the first error98                self._state = GzipDecoderState.SWALLOW_DATA99                if previous_state == GzipDecoderState.OTHER_MEMBERS:100                    # Allow trailing garbage acceptable in other gzip clients101                    return bytes(ret)102                raise103            data = self._obj.unused_data104            if not data:105                return bytes(ret)106            self._state = GzipDecoderState.OTHER_MEMBERS107            self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)108 109 110if brotli is not None:111 112    class BrotliDecoder(object):113        # Supports both 'brotlipy' and 'Brotli' packages114        # since they share an import name. The top branches115        # are for 'brotlipy' and bottom branches for 'Brotli'116        def __init__(self):117            self._obj = brotli.Decompressor()118            if hasattr(self._obj, "decompress"):119                self.decompress = self._obj.decompress120            else:121                self.decompress = self._obj.process122 123        def flush(self):124            if hasattr(self._obj, "flush"):125                return self._obj.flush()126            return b""127 128 129class MultiDecoder(object):130    """131    From RFC7231:132        If one or more encodings have been applied to a representation, the133        sender that applied the encodings MUST generate a Content-Encoding134        header field that lists the content codings in the order in which135        they were applied.136    """137 138    def __init__(self, modes):139        self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")]140 141    def flush(self):142        return self._decoders[0].flush()143 144    def decompress(self, data):145        for d in reversed(self._decoders):146            data = d.decompress(data)147        return data148 149 150def _get_decoder(mode):151    if "," in mode:152        return MultiDecoder(mode)153 154    if mode == "gzip":155        return GzipDecoder()156 157    if brotli is not None and mode == "br":158        return BrotliDecoder()159 160    return DeflateDecoder()161 162 163class HTTPResponse(io.IOBase):164    """165    HTTP Response container.166 167    Backwards-compatible with :class:`http.client.HTTPResponse` but the response ``body`` is168    loaded and decoded on-demand when the ``data`` property is accessed.  This169    class is also compatible with the Python standard library's :mod:`io`170    module, and can hence be treated as a readable object in the context of that171    framework.172 173    Extra parameters for behaviour not present in :class:`http.client.HTTPResponse`:174 175    :param preload_content:176        If True, the response's body will be preloaded during construction.177 178    :param decode_content:179        If True, will attempt to decode the body based on the180        'content-encoding' header.181 182    :param original_response:183        When this HTTPResponse wrapper is generated from an :class:`http.client.HTTPResponse`184        object, it's convenient to include the original for debug purposes. It's185        otherwise unused.186 187    :param retries:188        The retries contains the last :class:`~urllib3.util.retry.Retry` that189        was used during the request.190 191    :param enforce_content_length:192        Enforce content length checking. Body returned by server must match193        value of Content-Length header, if present. Otherwise, raise error.194    """195 196    CONTENT_DECODERS = ["gzip", "deflate"]197    if brotli is not None:198        CONTENT_DECODERS += ["br"]199    REDIRECT_STATUSES = [301, 302, 303, 307, 308]200 201    def __init__(202        self,203        body="",204        headers=None,205        status=0,206        version=0,207        reason=None,208        strict=0,209        preload_content=True,210        decode_content=True,211        original_response=None,212        pool=None,213        connection=None,214        msg=None,215        retries=None,216        enforce_content_length=False,217        request_method=None,218        request_url=None,219        auto_close=True,220    ):221 222        if isinstance(headers, HTTPHeaderDict):223            self.headers = headers224        else:225            self.headers = HTTPHeaderDict(headers)226        self.status = status227        self.version = version228        self.reason = reason229        self.strict = strict230        self.decode_content = decode_content231        self.retries = retries232        self.enforce_content_length = enforce_content_length233        self.auto_close = auto_close234 235        self._decoder = None236        self._body = None237        self._fp = None238        self._original_response = original_response239        self._fp_bytes_read = 0240        self.msg = msg241        self._request_url = request_url242 243        if body and isinstance(body, (six.string_types, bytes)):244            self._body = body245 246        self._pool = pool247        self._connection = connection248 249        if hasattr(body, "read"):250            self._fp = body251 252        # Are we using the chunked-style of transfer encoding?253        self.chunked = False254        self.chunk_left = None255        tr_enc = self.headers.get("transfer-encoding", "").lower()256        # Don't incur the penalty of creating a list and then discarding it257        encodings = (enc.strip() for enc in tr_enc.split(","))258        if "chunked" in encodings:259            self.chunked = True260 261        # Determine length of response262        self.length_remaining = self._init_length(request_method)263 264        # If requested, preload the body.265        if preload_content and not self._body:266            self._body = self.read(decode_content=decode_content)267 268    def get_redirect_location(self):269        """270        Should we redirect and where to?271 272        :returns: Truthy redirect location string if we got a redirect status273            code and valid location. ``None`` if redirect status and no274            location. ``False`` if not a redirect status code.275        """276        if self.status in self.REDIRECT_STATUSES:277            return self.headers.get("location")278 279        return False280 281    def release_conn(self):282        if not self._pool or not self._connection:283            return284 285        self._pool._put_conn(self._connection)286        self._connection = None287 288    def drain_conn(self):289        """290        Read and discard any remaining HTTP response data in the response connection.291 292        Unread data in the HTTPResponse connection blocks the connection from being released back to the pool.293        """294        try:295            self.read()296        except (HTTPError, SocketError, BaseSSLError, HTTPException):297            pass298 299    @property300    def data(self):301        # For backwards-compat with earlier urllib3 0.4 and earlier.302        if self._body:303            return self._body304 305        if self._fp:306            return self.read(cache_content=True)307 308    @property309    def connection(self):310        return self._connection311 312    def isclosed(self):313        return is_fp_closed(self._fp)314 315    def tell(self):316        """317        Obtain the number of bytes pulled over the wire so far. May differ from318        the amount of content returned by :meth:``urllib3.response.HTTPResponse.read``319        if bytes are encoded on the wire (e.g, compressed).320        """321        return self._fp_bytes_read322 323    def _init_length(self, request_method):324        """325        Set initial length value for Response content if available.326        """327        length = self.headers.get("content-length")328 329        if length is not None:330            if self.chunked:331                # This Response will fail with an IncompleteRead if it can't be332                # received as chunked. This method falls back to attempt reading333                # the response before raising an exception.334                log.warning(335                    "Received response with both Content-Length and "336                    "Transfer-Encoding set. This is expressly forbidden "337                    "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "338                    "attempting to process response as Transfer-Encoding: "339                    "chunked."340                )341                return None342 343            try:344                # RFC 7230 section 3.3.2 specifies multiple content lengths can345                # be sent in a single Content-Length header346                # (e.g. Content-Length: 42, 42). This line ensures the values347                # are all valid ints and that as long as the `set` length is 1,348                # all values are the same. Otherwise, the header is invalid.349                lengths = set([int(val) for val in length.split(",")])350                if len(lengths) > 1:351                    raise InvalidHeader(352                        "Content-Length contained multiple "353                        "unmatching values (%s)" % length354                    )355                length = lengths.pop()356            except ValueError:357                length = None358            else:359                if length < 0:360                    length = None361 362        # Convert status to int for comparison363        # In some cases, httplib returns a status of "_UNKNOWN"364        try:365            status = int(self.status)366        except ValueError:367            status = 0368 369        # Check for responses that shouldn't include a body370        if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":371            length = 0372 373        return length374 375    def _init_decoder(self):376        """377        Set-up the _decoder attribute if necessary.378        """379        # Note: content-encoding value should be case-insensitive, per RFC 7230380        # Section 3.2381        content_encoding = self.headers.get("content-encoding", "").lower()382        if self._decoder is None:383            if content_encoding in self.CONTENT_DECODERS:384                self._decoder = _get_decoder(content_encoding)385            elif "," in content_encoding:386                encodings = [387                    e.strip()388                    for e in content_encoding.split(",")389                    if e.strip() in self.CONTENT_DECODERS390                ]391                if len(encodings):392                    self._decoder = _get_decoder(content_encoding)393 394    DECODER_ERROR_CLASSES = (IOError, zlib.error)395    if brotli is not None:396        DECODER_ERROR_CLASSES += (brotli.error,)397 398    def _decode(self, data, decode_content, flush_decoder):399        """400        Decode the data passed in and potentially flush the decoder.401        """402        if not decode_content:403            return data404 405        try:406            if self._decoder:407                data = self._decoder.decompress(data)408        except self.DECODER_ERROR_CLASSES as e:409            content_encoding = self.headers.get("content-encoding", "").lower()410            raise DecodeError(411                "Received response with content-encoding: %s, but "412                "failed to decode it." % content_encoding,413                e,414            )415        if flush_decoder:416            data += self._flush_decoder()417 418        return data419 420    def _flush_decoder(self):421        """422        Flushes the decoder. Should only be called if the decoder is actually423        being used.424        """425        if self._decoder:426            buf = self._decoder.decompress(b"")427            return buf + self._decoder.flush()428 429        return b""430 431    @contextmanager432    def _error_catcher(self):433        """434        Catch low-level python exceptions, instead re-raising urllib3435        variants, so that low-level exceptions are not leaked in the436        high-level api.437 438        On exit, release the connection back to the pool.439        """440        clean_exit = False441 442        try:443            try:444                yield445 446            except SocketTimeout:447                # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but448                # there is yet no clean way to get at it from this context.449                raise ReadTimeoutError(self._pool, None, "Read timed out.")450 451            except BaseSSLError as e:452                # FIXME: Is there a better way to differentiate between SSLErrors?453                if "read operation timed out" not in str(e):454                    # SSL errors related to framing/MAC get wrapped and reraised here455                    raise SSLError(e)456 457                raise ReadTimeoutError(self._pool, None, "Read timed out.")458 459            except (HTTPException, SocketError) as e:460                # This includes IncompleteRead.461                raise ProtocolError("Connection broken: %r" % e, e)462 463            # If no exception is thrown, we should avoid cleaning up464            # unnecessarily.465            clean_exit = True466        finally:467            # If we didn't terminate cleanly, we need to throw away our468            # connection.469            if not clean_exit:470                # The response may not be closed but we're not going to use it471                # anymore so close it now to ensure that the connection is472                # released back to the pool.473                if self._original_response:474                    self._original_response.close()475 476                # Closing the response may not actually be sufficient to close477                # everything, so if we have a hold of the connection close that478                # too.479                if self._connection:480                    self._connection.close()481 482            # If we hold the original response but it's closed now, we should483            # return the connection back to the pool.484            if self._original_response and self._original_response.isclosed():485                self.release_conn()486 487    def _fp_read(self, amt):488        """489        Read a response with the thought that reading the number of bytes490        larger than can fit in a 32-bit int at a time via SSL in some491        known cases leads to an overflow error that has to be prevented492        if `amt` or `self.length_remaining` indicate that a problem may493        happen.494 495        The known cases:496          * 3.8 <= CPython < 3.9.7 because of a bug497            https://github.com/urllib3/urllib3/issues/2513#issuecomment-1152559900.498          * urllib3 injected with pyOpenSSL-backed SSL-support.499          * CPython < 3.10 only when `amt` does not fit 32-bit int.500        """501        assert self._fp502        c_int_max = 2 ** 31 - 1503        if (504            (505                (amt and amt > c_int_max)506                or (self.length_remaining and self.length_remaining > c_int_max)507            )508            and not util.IS_SECURETRANSPORT509            and (util.IS_PYOPENSSL or sys.version_info < (3, 10))510        ):511            buffer = io.BytesIO()512            # Besides `max_chunk_amt` being a maximum chunk size, it513            # affects memory overhead of reading a response by this514            # method in CPython.515            # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum516            # chunk size that does not lead to an overflow error, but517            # 256 MiB is a compromise.518            max_chunk_amt = 2 ** 28519            while amt is None or amt != 0:520                if amt is not None:521                    chunk_amt = min(amt, max_chunk_amt)522                    amt -= chunk_amt523                else:524                    chunk_amt = max_chunk_amt525                data = self._fp.read(chunk_amt)526                if not data:527                    break528                buffer.write(data)529                del data  # to reduce peak memory usage by `max_chunk_amt`.530            return buffer.getvalue()531        else:532            # StringIO doesn't like amt=None533            return self._fp.read(amt) if amt is not None else self._fp.read()534 535    def read(self, amt=None, decode_content=None, cache_content=False):536        """537        Similar to :meth:`http.client.HTTPResponse.read`, but with two additional538        parameters: ``decode_content`` and ``cache_content``.539 540        :param amt:541            How much of the content to read. If specified, caching is skipped542            because it doesn't make sense to cache partial content as the full543            response.544 545        :param decode_content:546            If True, will attempt to decode the body based on the547            'content-encoding' header.548 549        :param cache_content:550            If True, will save the returned data such that the same result is551            returned despite of the state of the underlying file object. This552            is useful if you want the ``.data`` property to continue working553            after having ``.read()`` the file object. (Overridden if ``amt`` is554            set.)555        """556        self._init_decoder()557        if decode_content is None:558            decode_content = self.decode_content559 560        if self._fp is None:561            return562 563        flush_decoder = False564        fp_closed = getattr(self._fp, "closed", False)565 566        with self._error_catcher():567            data = self._fp_read(amt) if not fp_closed else b""568            if amt is None:569                flush_decoder = True570            else:571                cache_content = False572                if (573                    amt != 0 and not data574                ):  # Platform-specific: Buggy versions of Python.575                    # Close the connection when no data is returned576                    #577                    # This is redundant to what httplib/http.client _should_578                    # already do.  However, versions of python released before579                    # December 15, 2012 (http://bugs.python.org/issue16298) do580                    # not properly close the connection in all cases. There is581                    # no harm in redundantly calling close.582                    self._fp.close()583                    flush_decoder = True584                    if self.enforce_content_length and self.length_remaining not in (585                        0,586                        None,587                    ):588                        # This is an edge case that httplib failed to cover due589                        # to concerns of backward compatibility. We're590                        # addressing it here to make sure IncompleteRead is591                        # raised during streaming, so all calls with incorrect592                        # Content-Length are caught.593                        raise IncompleteRead(self._fp_bytes_read, self.length_remaining)594 595        if data:596            self._fp_bytes_read += len(data)597            if self.length_remaining is not None:598                self.length_remaining -= len(data)599 600            data = self._decode(data, decode_content, flush_decoder)601 602            if cache_content:603                self._body = data604 605        return data606 607    def stream(self, amt=2 ** 16, decode_content=None):608        """609        A generator wrapper for the read() method. A call will block until610        ``amt`` bytes have been read from the connection or until the611        connection is closed.612 613        :param amt:614            How much of the content to read. The generator will return up to615            much data per iteration, but may return less. This is particularly616            likely when using compressed data. However, the empty string will617            never be returned.618 619        :param decode_content:620            If True, will attempt to decode the body based on the621            'content-encoding' header.622        """623        if self.chunked and self.supports_chunked_reads():624            for line in self.read_chunked(amt, decode_content=decode_content):625                yield line626        else:627            while not is_fp_closed(self._fp):628                data = self.read(amt=amt, decode_content=decode_content)629 630                if data:631                    yield data632 633    @classmethod634    def from_httplib(ResponseCls, r, **response_kw):635        """636        Given an :class:`http.client.HTTPResponse` instance ``r``, return a637        corresponding :class:`urllib3.response.HTTPResponse` object.638 639        Remaining parameters are passed to the HTTPResponse constructor, along640        with ``original_response=r``.641        """642        headers = r.msg643 644        if not isinstance(headers, HTTPHeaderDict):645            if six.PY2:646                # Python 2.7647                headers = HTTPHeaderDict.from_httplib(headers)648            else:649                headers = HTTPHeaderDict(headers.items())650 651        # HTTPResponse objects in Python 3 don't have a .strict attribute652        strict = getattr(r, "strict", 0)653        resp = ResponseCls(654            body=r,655            headers=headers,656            status=r.status,657            version=r.version,658            reason=r.reason,659            strict=strict,660            original_response=r,661            **response_kw662        )663        return resp664 665    # Backwards-compatibility methods for http.client.HTTPResponse666    def getheaders(self):667        warnings.warn(668            "HTTPResponse.getheaders() is deprecated and will be removed "669            "in urllib3 v2.1.0. Instead access HTTPResponse.headers directly.",670            category=DeprecationWarning,671            stacklevel=2,672        )673        return self.headers674 675    def getheader(self, name, default=None):676        warnings.warn(677            "HTTPResponse.getheader() is deprecated and will be removed "678            "in urllib3 v2.1.0. Instead use HTTPResponse.headers.get(name, default).",679            category=DeprecationWarning,680            stacklevel=2,681        )682        return self.headers.get(name, default)683 684    # Backwards compatibility for http.cookiejar685    def info(self):686        return self.headers687 688    # Overrides from io.IOBase689    def close(self):690        if not self.closed:691            self._fp.close()692 693        if self._connection:694            self._connection.close()695 696        if not self.auto_close:697            io.IOBase.close(self)698 699    @property700    def closed(self):701        if not self.auto_close:702            return io.IOBase.closed.__get__(self)703        elif self._fp is None:704            return True705        elif hasattr(self._fp, "isclosed"):706            return self._fp.isclosed()707        elif hasattr(self._fp, "closed"):708            return self._fp.closed709        else:710            return True711 712    def fileno(self):713        if self._fp is None:714            raise IOError("HTTPResponse has no file to get a fileno from")715        elif hasattr(self._fp, "fileno"):716            return self._fp.fileno()717        else:718            raise IOError(719                "The file-like object this HTTPResponse is wrapped "720                "around has no file descriptor"721            )722 723    def flush(self):724        if (725            self._fp is not None726            and hasattr(self._fp, "flush")727            and not getattr(self._fp, "closed", False)728        ):729            return self._fp.flush()730 731    def readable(self):732        # This method is required for `io` module compatibility.733        return True734 735    def readinto(self, b):736        # This method is required for `io` module compatibility.737        temp = self.read(len(b))738        if len(temp) == 0:739            return 0740        else:741            b[: len(temp)] = temp742            return len(temp)743 744    def supports_chunked_reads(self):745        """746        Checks if the underlying file-like object looks like a747        :class:`http.client.HTTPResponse` object. We do this by testing for748        the fp attribute. If it is present we assume it returns raw chunks as749        processed by read_chunked().750        """751        return hasattr(self._fp, "fp")752 753    def _update_chunk_length(self):754        # First, we'll figure out length of a chunk and then755        # we'll try to read it from socket.756        if self.chunk_left is not None:757            return758        line = self._fp.fp.readline()759        line = line.split(b";", 1)[0]760        try:761            self.chunk_left = int(line, 16)762        except ValueError:763            # Invalid chunked protocol response, abort.764            self.close()765            raise InvalidChunkLength(self, line)766 767    def _handle_chunk(self, amt):768        returned_chunk = None769        if amt is None:770            chunk = self._fp._safe_read(self.chunk_left)771            returned_chunk = chunk772            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.773            self.chunk_left = None774        elif amt < self.chunk_left:775            value = self._fp._safe_read(amt)776            self.chunk_left = self.chunk_left - amt777            returned_chunk = value778        elif amt == self.chunk_left:779            value = self._fp._safe_read(amt)780            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.781            self.chunk_left = None782            returned_chunk = value783        else:  # amt > self.chunk_left784            returned_chunk = self._fp._safe_read(self.chunk_left)785            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.786            self.chunk_left = None787        return returned_chunk788 789    def read_chunked(self, amt=None, decode_content=None):790        """791        Similar to :meth:`HTTPResponse.read`, but with an additional792        parameter: ``decode_content``.793 794        :param amt:795            How much of the content to read. If specified, caching is skipped796            because it doesn't make sense to cache partial content as the full797            response.798 799        :param decode_content:800            If True, will attempt to decode the body based on the801            'content-encoding' header.802        """803        self._init_decoder()804        # FIXME: Rewrite this method and make it a class with a better structured logic.805        if not self.chunked:806            raise ResponseNotChunked(807                "Response is not chunked. "808                "Header 'transfer-encoding: chunked' is missing."809            )810        if not self.supports_chunked_reads():811            raise BodyNotHttplibCompatible(812                "Body should be http.client.HTTPResponse like. "813                "It should have have an fp attribute which returns raw chunks."814            )815 816        with self._error_catcher():817            # Don't bother reading the body of a HEAD request.818            if self._original_response and is_response_to_head(self._original_response):819                self._original_response.close()820                return821 822            # If a response is already read and closed823            # then return immediately.824            if self._fp.fp is None:825                return826 827            while True:828                self._update_chunk_length()829                if self.chunk_left == 0:830                    break831                chunk = self._handle_chunk(amt)832                decoded = self._decode(833                    chunk, decode_content=decode_content, flush_decoder=False834                )835                if decoded:836                    yield decoded837 838            if decode_content:839                # On CPython and PyPy, we should never need to flush the840                # decoder. However, on Jython we *might* need to, so841                # lets defensively do it anyway.842                decoded = self._flush_decoder()843                if decoded:  # Platform-specific: Jython.844                    yield decoded845 846            # Chunk content ends with \r\n: discard it.847            while True:848                line = self._fp.fp.readline()849                if not line:850                    # Some sites may not end with '\r\n'.851                    break852                if line == b"\r\n":853                    break854 855            # We read everything; close the "file".856            if self._original_response:857                self._original_response.close()858 859    def geturl(self):860        """861        Returns the URL that was the source of this response.862        If the request that generated this response redirected, this method863        will return the final redirect location.864        """865        if self.retries is not None and len(self.retries.history):866            return self.retries.history[-1].redirect_location867        else:868            return self._request_url869 870    def __iter__(self):871        buffer = []872        for chunk in self.stream(decode_content=True):873            if b"\n" in chunk:874                chunk = chunk.split(b"\n")875                yield b"".join(buffer) + chunk[0] + b"\n"876                for x in chunk[1:-1]:877                    yield x + b"\n"878                if chunk[-1]:879                    buffer = [chunk[-1]]880                else:881                    buffer = []882            else:883                buffer.append(chunk)884        if buffer:885            yield b"".join(buffer)886