codekingpro/portable-devtools

1# Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/2# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License"). You5# may not use this file except in compliance with the License. A copy of6# the License is located at7#8# http://aws.amazon.com/apache2.0/9#10# or in the "license" file accompanying this file. This file is11# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF12# ANY KIND, either express or implied. See the License for the specific13# language governing permissions and limitations under the License.14import functools15import logging16from collections.abc import Mapping17 18import urllib3.util19from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection20from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool21 22import botocore.utils23from botocore.compat import (24    HTTPHeaders,25    HTTPResponse,26    MutableMapping,27    urlencode,28    urlparse,29    urlsplit,30    urlunsplit,31)32from botocore.exceptions import UnseekableStreamError33 34logger = logging.getLogger(__name__)35 36 37class AWSHTTPResponse(HTTPResponse):38    # The *args, **kwargs is used because the args are slightly39    # different in py2.6 than in py2.7/py3.40    def __init__(self, *args, **kwargs):41        self._status_tuple = kwargs.pop('status_tuple')42        HTTPResponse.__init__(self, *args, **kwargs)43 44    def _read_status(self):45        if self._status_tuple is not None:46            status_tuple = self._status_tuple47            self._status_tuple = None48            return status_tuple49        else:50            return HTTPResponse._read_status(self)51 52 53class AWSConnection:54    """Mixin for HTTPConnection that supports Expect 100-continue.55 56    This when mixed with a subclass of httplib.HTTPConnection (though57    technically we subclass from urllib3, which subclasses58    httplib.HTTPConnection) and we only override this class to support Expect59    100-continue, which we need for S3.  As far as I can tell, this is60    general purpose enough to not be specific to S3, but I'm being61    tentative and keeping it in botocore because I've only tested62    this against AWS services.63 64    """65 66    def __init__(self, *args, **kwargs):67        super().__init__(*args, **kwargs)68        self._original_response_cls = self.response_class69        # This variable is set when we receive an early response from the70        # server. If this value is set to True, any calls to send() are noops.71        # This value is reset to false every time _send_request is called.72        # This is to workaround changes in urllib3 2.0 which uses separate73        # send() calls in request() instead of delegating to endheaders(),74        # which is where the body is sent in CPython's HTTPConnection.75        self._response_received = False76        self._expect_header_set = False77        self._send_called = False78 79    def close(self):80        super().close()81        # Reset all of our instance state we were tracking.82        self._response_received = False83        self._expect_header_set = False84        self._send_called = False85        self.response_class = self._original_response_cls86 87    def request(self, method, url, body=None, headers=None, *args, **kwargs):88        if headers is None:89            headers = {}90        self._response_received = False91        if headers.get('Expect', b'') == b'100-continue':92            self._expect_header_set = True93        else:94            self._expect_header_set = False95            self.response_class = self._original_response_cls96        rval = super().request(method, url, body, headers, *args, **kwargs)97        self._expect_header_set = False98        return rval99 100    def _convert_to_bytes(self, mixed_buffer):101        # Take a list of mixed str/bytes and convert it102        # all into a single bytestring.103        # Any str will be encoded as utf-8.104        bytes_buffer = []105        for chunk in mixed_buffer:106            if isinstance(chunk, str):107                bytes_buffer.append(chunk.encode('utf-8'))108            else:109                bytes_buffer.append(chunk)110        msg = b"\r\n".join(bytes_buffer)111        return msg112 113    def _send_output(self, message_body=None, *args, **kwargs):114        self._buffer.extend((b"", b""))115        msg = self._convert_to_bytes(self._buffer)116        del self._buffer[:]117        # If msg and message_body are sent in a single send() call,118        # it will avoid performance problems caused by the interaction119        # between delayed ack and the Nagle algorithm.120        if isinstance(message_body, bytes):121            msg += message_body122            message_body = None123        self.send(msg)124        if self._expect_header_set:125            # This is our custom behavior.  If the Expect header was126            # set, it will trigger this custom behavior.127            logger.debug("Waiting for 100 Continue response.")128            # Wait for 1 second for the server to send a response.129            if urllib3.util.wait_for_read(self.sock, 1):130                self._handle_expect_response(message_body)131                return132            else:133                # From the RFC:134                # Because of the presence of older implementations, the135                # protocol allows ambiguous situations in which a client may136                # send "Expect: 100-continue" without receiving either a 417137                # (Expectation Failed) status or a 100 (Continue) status.138                # Therefore, when a client sends this header field to an origin139                # server (possibly via a proxy) from which it has never seen a140                # 100 (Continue) status, the client SHOULD NOT wait for an141                # indefinite period before sending the request body.142                logger.debug(143                    "No response seen from server, continuing to "144                    "send the response body."145                )146        if message_body is not None:147            # message_body was not a string (i.e. it is a file), and148            # we must run the risk of Nagle.149            self.send(message_body)150 151    def _consume_headers(self, fp):152        # Most servers (including S3) will just return153        # the CLRF after the 100 continue response.  However,154        # some servers (I've specifically seen this for squid when155        # used as a straight HTTP proxy) will also inject a156        # Connection: keep-alive header.  To account for this157        # we'll read until we read '\r\n', and ignore any headers158        # that come immediately after the 100 continue response.159        current = None160        while current != b'\r\n':161            current = fp.readline()162 163    def _handle_expect_response(self, message_body):164        # This is called when we sent the request headers containing165        # an Expect: 100-continue header and received a response.166        # We now need to figure out what to do.167        fp = self.sock.makefile('rb', 0)168        try:169            maybe_status_line = fp.readline()170            parts = maybe_status_line.split(None, 2)171            if self._is_100_continue_status(maybe_status_line):172                self._consume_headers(fp)173                logger.debug(174                    "100 Continue response seen, now sending request body."175                )176                self._send_message_body(message_body)177            elif len(parts) == 3 and parts[0].startswith(b'HTTP/'):178                # From the RFC:179                # Requirements for HTTP/1.1 origin servers:180                #181                # - Upon receiving a request which includes an Expect182                #   request-header field with the "100-continue"183                #   expectation, an origin server MUST either respond with184                #   100 (Continue) status and continue to read from the185                #   input stream, or respond with a final status code.186                #187                # So if we don't get a 100 Continue response, then188                # whatever the server has sent back is the final response189                # and don't send the message_body.190                logger.debug(191                    "Received a non 100 Continue response "192                    "from the server, NOT sending request body."193                )194                status_tuple = (195                    parts[0].decode('ascii'),196                    int(parts[1]),197                    parts[2].decode('ascii'),198                )199                response_class = functools.partial(200                    AWSHTTPResponse, status_tuple=status_tuple201                )202                self.response_class = response_class203                self._response_received = True204        finally:205            fp.close()206 207    def _send_message_body(self, message_body):208        if message_body is not None:209            self.send(message_body)210 211    def send(self, str):212        if self._response_received:213            if not self._send_called:214                # urllib3 2.0 chunks and calls send potentially215                # thousands of times inside `request` unlike the216                # standard library. Only log this once for sanity.217                logger.debug(218                    "send() called, but response already received. "219                    "Not sending data."220                )221            self._send_called = True222            return223        return super().send(str)224 225    def _is_100_continue_status(self, maybe_status_line):226        parts = maybe_status_line.split(None, 2)227        # Check for HTTP/<version> 100 Continue\r\n228        return (229            len(parts) >= 3230            and parts[0].startswith(b'HTTP/')231            and parts[1] == b'100'232        )233 234 235class AWSHTTPConnection(AWSConnection, HTTPConnection):236    """An HTTPConnection that supports 100 Continue behavior."""237 238 239class AWSHTTPSConnection(AWSConnection, VerifiedHTTPSConnection):240    """An HTTPSConnection that supports 100 Continue behavior."""241 242 243class AWSHTTPConnectionPool(HTTPConnectionPool):244    ConnectionCls = AWSHTTPConnection245 246 247class AWSHTTPSConnectionPool(HTTPSConnectionPool):248    ConnectionCls = AWSHTTPSConnection249 250 251def prepare_request_dict(252    request_dict, endpoint_url, context=None, user_agent=None253):254    """255    This method prepares a request dict to be created into an256    AWSRequestObject. This prepares the request dict by adding the257    url and the user agent to the request dict.258 259    :type request_dict: dict260    :param request_dict:  The request dict (created from the261        ``serialize`` module).262 263    :type user_agent: string264    :param user_agent: The user agent to use for this request.265 266    :type endpoint_url: string267    :param endpoint_url: The full endpoint url, which contains at least268        the scheme, the hostname, and optionally any path components.269    """270    r = request_dict271    if user_agent is not None:272        headers = r['headers']273        headers['User-Agent'] = user_agent274    host_prefix = r.get('host_prefix')275    url = _urljoin(endpoint_url, r['url_path'], host_prefix)276    if r['query_string']:277        # NOTE: This is to avoid circular import with utils. This is being278        # done to avoid moving classes to different modules as to not cause279        # breaking chainges.280        percent_encode_sequence = botocore.utils.percent_encode_sequence281        encoded_query_string = percent_encode_sequence(r['query_string'])282        if '?' not in url:283            url += '?%s' % encoded_query_string284        else:285            url += '&%s' % encoded_query_string286    r['url'] = url287    r['context'] = context288    if context is None:289        r['context'] = {}290 291 292def create_request_object(request_dict):293    """294    This method takes a request dict and creates an AWSRequest object295    from it.296 297    :type request_dict: dict298    :param request_dict:  The request dict (created from the299        ``prepare_request_dict`` method).300 301    :rtype: ``botocore.awsrequest.AWSRequest``302    :return: An AWSRequest object based on the request_dict.303 304    """305    r = request_dict306    request_object = AWSRequest(307        method=r['method'],308        url=r['url'],309        data=r['body'],310        headers=r['headers'],311        auth_path=r.get('auth_path'),312    )313    request_object.context = r['context']314    return request_object315 316 317def _urljoin(endpoint_url, url_path, host_prefix):318    p = urlsplit(endpoint_url)319    # <part>   - <index>320    # scheme   - p[0]321    # netloc   - p[1]322    # path     - p[2]323    # query    - p[3]324    # fragment - p[4]325    if not url_path or url_path == '/':326        # If there's no path component, ensure the URL ends with327        # a '/' for backwards compatibility.328        if not p[2]:329            new_path = '/'330        else:331            new_path = p[2]332    elif p[2].endswith('/') and url_path.startswith('/'):333        new_path = p[2][:-1] + url_path334    else:335        new_path = p[2] + url_path336 337    new_netloc = p[1]338    if host_prefix is not None:339        new_netloc = host_prefix + new_netloc340 341    reconstructed = urlunsplit((p[0], new_netloc, new_path, p[3], p[4]))342    return reconstructed343 344 345class AWSRequestPreparer:346    """347    This class performs preparation on AWSRequest objects similar to that of348    the PreparedRequest class does in the requests library. However, the logic349    has been boiled down to meet the specific use cases in botocore. Of note350    there are the following differences:351        This class does not heavily prepare the URL. Requests performed many352        validations and corrections to ensure the URL is properly formatted.353        Botocore either performs these validations elsewhere or otherwise354        consistently provides well formatted URLs.355 356        This class does not heavily prepare the body. Body preperation is357        simple and supports only the cases that we document: bytes and358        file-like objects to determine the content-length. This will also359        additionally prepare a body that is a dict to be url encoded params360        string as some signers rely on this. Finally, this class does not361        support multipart file uploads.362 363        This class does not prepare the method, auth or cookies.364    """365 366    def prepare(self, original):367        method = original.method368        url = self._prepare_url(original)369        body = self._prepare_body(original)370        headers = self._prepare_headers(original, body)371        stream_output = original.stream_output372 373        return AWSPreparedRequest(method, url, headers, body, stream_output)374 375    def _prepare_url(self, original):376        url = original.url377        if original.params:378            url_parts = urlparse(url)379            delim = '&' if url_parts.query else '?'380            if isinstance(original.params, Mapping):381                params_to_encode = list(original.params.items())382            else:383                params_to_encode = original.params384            params = urlencode(params_to_encode, doseq=True)385            url = delim.join((url, params))386        return url387 388    def _prepare_headers(self, original, prepared_body=None):389        headers = HeadersDict(original.headers.items())390 391        # If the transfer encoding or content length is already set, use that392        if 'Transfer-Encoding' in headers or 'Content-Length' in headers:393            return headers394 395        # Ensure we set the content length when it is expected396        if original.method not in ('GET', 'HEAD', 'OPTIONS'):397            length = self._determine_content_length(prepared_body)398            if length is not None:399                headers['Content-Length'] = str(length)400            else:401                # Failed to determine content length, using chunked402                # NOTE: This shouldn't ever happen in practice403                body_type = type(prepared_body)404                logger.debug('Failed to determine length of %s', body_type)405                headers['Transfer-Encoding'] = 'chunked'406 407        return headers408 409    def _to_utf8(self, item):410        key, value = item411        if isinstance(key, str):412            key = key.encode('utf-8')413        if isinstance(value, str):414            value = value.encode('utf-8')415        return key, value416 417    def _prepare_body(self, original):418        """Prepares the given HTTP body data."""419        body = original.data420        if body == b'':421            body = None422 423        if isinstance(body, dict):424            params = [self._to_utf8(item) for item in body.items()]425            body = urlencode(params, doseq=True)426 427        return body428 429    def _determine_content_length(self, body):430        return botocore.utils.determine_content_length(body)431 432 433class AWSRequest:434    """Represents the elements of an HTTP request.435 436    This class was originally inspired by requests.models.Request, but has been437    boiled down to meet the specific use cases in botocore. That being said this438    class (even in requests) is effectively a named-tuple.439    """440 441    _REQUEST_PREPARER_CLS = AWSRequestPreparer442 443    def __init__(444        self,445        method=None,446        url=None,447        headers=None,448        data=None,449        params=None,450        auth_path=None,451        stream_output=False,452    ):453        self._request_preparer = self._REQUEST_PREPARER_CLS()454 455        # Default empty dicts for dict params.456        params = {} if params is None else params457 458        self.method = method459        self.url = url460        self.headers = HTTPHeaders()461        self.data = data462        self.params = params463        self.auth_path = auth_path464        self.stream_output = stream_output465 466        if headers is not None:467            for key, value in headers.items():468                self.headers[key] = value469 470        # This is a dictionary to hold information that is used when471        # processing the request. What is inside of ``context`` is open-ended.472        # For example, it may have a timestamp key that is used for holding473        # what the timestamp is when signing the request. Note that none474        # of the information that is inside of ``context`` is directly475        # sent over the wire; the information is only used to assist in476        # creating what is sent over the wire.477        self.context = {}478 479    def prepare(self):480        """Constructs a :class:`AWSPreparedRequest <AWSPreparedRequest>`."""481        return self._request_preparer.prepare(self)482 483    @property484    def body(self):485        body = self.prepare().body486        if isinstance(body, str):487            body = body.encode('utf-8')488        return body489 490 491class AWSPreparedRequest:492    """A data class representing a finalized request to be sent over the wire.493 494    Requests at this stage should be treated as final, and the properties of495    the request should not be modified.496 497    :ivar method: The HTTP Method498    :ivar url: The full url499    :ivar headers: The HTTP headers to send.500    :ivar body: The HTTP body.501    :ivar stream_output: If the response for this request should be streamed.502    """503 504    def __init__(self, method, url, headers, body, stream_output):505        self.method = method506        self.url = url507        self.headers = headers508        self.body = body509        self.stream_output = stream_output510 511    def __repr__(self):512        fmt = (513            '<AWSPreparedRequest stream_output=%s, method=%s, url=%s, '514            'headers=%s>'515        )516        return fmt % (self.stream_output, self.method, self.url, self.headers)517 518    def reset_stream(self):519        """Resets the streaming body to it's initial position.520 521        If the request contains a streaming body (a streamable file-like object)522        seek to the object's initial position to ensure the entire contents of523        the object is sent. This is a no-op for static bytes-like body types.524        """525        # Trying to reset a stream when there is a no stream will526        # just immediately return.  It's not an error, it will produce527        # the same result as if we had actually reset the stream (we'll send528        # the entire body contents again if we need to).529        # Same case if the body is a string/bytes/bytearray type.530 531        non_seekable_types = (bytes, str, bytearray)532        if self.body is None or isinstance(self.body, non_seekable_types):533            return534        try:535            logger.debug("Rewinding stream: %s", self.body)536            self.body.seek(0)537        except Exception as e:538            logger.debug("Unable to rewind stream: %s", e)539            raise UnseekableStreamError(stream_object=self.body)540 541 542class AWSResponse:543    """A data class representing an HTTP response.544 545    This class was originally inspired by requests.models.Response, but has546    been boiled down to meet the specific use cases in botocore. This has547    effectively been reduced to a named tuple.548 549    :ivar url: The full url.550    :ivar status_code: The status code of the HTTP response.551    :ivar headers: The HTTP headers received.552    :ivar body: The HTTP response body.553    """554 555    def __init__(self, url, status_code, headers, raw):556        self.url = url557        self.status_code = status_code558        self.headers = HeadersDict(headers)559        self.raw = raw560 561        self._content = None562 563    @property564    def content(self):565        """Content of the response as bytes."""566 567        if self._content is None:568            # Read the contents.569            # NOTE: requests would attempt to call stream and fall back570            # to a custom generator that would call read in a loop, but571            # we don't rely on this behavior572            self._content = b''.join(self.raw.stream()) or b''573 574        return self._content575 576    @property577    def text(self):578        """Content of the response as a proper text type.579 580        Uses the encoding type provided in the reponse headers to decode the581        response content into a proper text type. If the encoding is not582        present in the headers, UTF-8 is used as a default.583        """584        encoding = botocore.utils.get_encoding_from_headers(self.headers)585        if encoding:586            return self.content.decode(encoding)587        else:588            return self.content.decode('utf-8')589 590 591class _HeaderKey:592    def __init__(self, key):593        self._key = key594        self._lower = key.lower()595 596    def __hash__(self):597        return hash(self._lower)598 599    def __eq__(self, other):600        return isinstance(other, _HeaderKey) and self._lower == other._lower601 602    def __str__(self):603        return self._key604 605    def __repr__(self):606        return repr(self._key)607 608 609class HeadersDict(MutableMapping):610    """A case-insenseitive dictionary to represent HTTP headers."""611 612    def __init__(self, *args, **kwargs):613        self._dict = {}614        self.update(*args, **kwargs)615 616    def __setitem__(self, key, value):617        self._dict[_HeaderKey(key)] = value618 619    def __getitem__(self, key):620        return self._dict[_HeaderKey(key)]621 622    def __delitem__(self, key):623        del self._dict[_HeaderKey(key)]624 625    def __iter__(self):626        return (str(key) for key in self._dict)627 628    def __len__(self):629        return len(self._dict)630 631    def __repr__(self):632        return repr(self._dict)633 634    def copy(self):635        return HeadersDict(self.items())636