codekingpro/portable-devtools

1# --------------------------------------------------------------------------2#3# Copyright (c) Microsoft Corporation. All rights reserved.4#5# The MIT License (MIT)6#7# Permission is hereby granted, free of charge, to any person obtaining a copy8# of this software and associated documentation files (the ""Software""), to9# deal in the Software without restriction, including without limitation the10# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or11# sell copies of the Software, and to permit persons to whom the Software is12# furnished to do so, subject to the following conditions:13#14# The above copyright notice and this permission notice shall be included in15# all copies or substantial portions of the Software.16#17# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR18# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,19# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE20# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER21# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING22# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS23# IN THE SOFTWARE.24#25# --------------------------------------------------------------------------26from __future__ import absolute_import  # we have a "requests" module that conflicts with "requests" on Py2.727import abc28try:29    import configparser30    from configparser import NoOptionError31except ImportError:32    import ConfigParser as configparser  # type: ignore33    from ConfigParser import NoOptionError  # type: ignore34import json35import logging36import os.path37try:38    from urlparse import urlparse39except ImportError:40    from urllib.parse import urlparse41import xml.etree.ElementTree as ET42 43from typing import TYPE_CHECKING, Generic, TypeVar, cast, IO, List, Union, Any, Mapping, Dict, Optional, Tuple, Callable, Iterator, MutableMapping  # pylint: disable=unused-import44 45HTTPResponseType = TypeVar("HTTPResponseType", bound='HTTPClientResponse')46 47# This file is NOT using any "requests" HTTP implementation48# However, the CaseInsensitiveDict is handy.49# If one day we reach the point where "requests" can be skip totally,50# might provide our own implementation51from requests.structures import CaseInsensitiveDict52 53from ..exceptions import ClientRequestError, raise_with_traceback54 55if TYPE_CHECKING:56    from ..serialization import Model  # pylint: disable=unused-import57 58 59_LOGGER = logging.getLogger(__name__)60 61try:62    ABC = abc.ABC63except AttributeError: # Python 2.7, abc exists, but not ABC64    ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()})  # type: ignore65 66try:67    from contextlib import AbstractContextManager  # type: ignore68except ImportError: # Python <= 3.569    class AbstractContextManager(object):  # type: ignore70        def __enter__(self):71            """Return `self` upon entering the runtime context."""72            return self73 74        @abc.abstractmethod75        def __exit__(self, exc_type, exc_value, traceback):76            """Raise any exception triggered within the runtime context."""77            return None78 79 80class HTTPSender(AbstractContextManager, ABC):81    """An http sender ABC.82    """83 84    @abc.abstractmethod85    def send(self, request, **config):86        # type: (ClientRequest, Any) -> ClientResponse87        """Send the request using this HTTP sender.88        """89        pass90 91 92class HTTPSenderConfiguration(object):93    """HTTP sender configuration.94 95    This is composed of generic HTTP configuration, and could be use as a common96    HTTP configuration format.97 98    :param str filepath: Path to existing config file (optional).99    """100 101    def __init__(self, filepath=None):102        # Communication configuration103        self.connection = ClientConnection()104 105        # Headers (sent with every requests)106        self.headers = {}  # type: Dict[str, str]107 108        # ProxyConfiguration109        self.proxies = ClientProxies()110 111        # Redirect configuration112        self.redirect_policy = ClientRedirectPolicy()113 114        self._config = configparser.ConfigParser()115        self._config.optionxform = str  # type: ignore116 117        if filepath:118            self.load(filepath)119 120    def _clear_config(self):121        # type: () -> None122        """Clearout config object in memory."""123        for section in self._config.sections():124            self._config.remove_section(section)125 126    def save(self, filepath):127        # type: (str) -> None128        """Save current configuration to file.129 130        :param str filepath: Path to file where settings will be saved.131        :raises: ValueError if supplied filepath cannot be written to.132        """133        sections = [134            "Connection",135            "Proxies",136            "RedirectPolicy"]137        for section in sections:138            self._config.add_section(section)139 140        self._config.set("Connection", "timeout", self.connection.timeout)141        self._config.set("Connection", "verify", self.connection.verify)142        self._config.set("Connection", "cert", self.connection.cert)143 144        self._config.set("Proxies", "proxies", self.proxies.proxies)145        self._config.set("Proxies", "env_settings",146                         self.proxies.use_env_settings)147 148        self._config.set("RedirectPolicy", "allow", self.redirect_policy.allow)149        self._config.set("RedirectPolicy", "max_redirects",150                         self.redirect_policy.max_redirects)151 152        try:153            with open(filepath, 'w') as configfile:154                self._config.write(configfile)155        except (KeyError, EnvironmentError):156            error = "Supplied config filepath invalid."157            raise_with_traceback(ValueError, error)158        finally:159            self._clear_config()160 161    def load(self, filepath):162        # type: (str) -> None163        """Load configuration from existing file.164 165        :param str filepath: Path to existing config file.166        :raises: ValueError if supplied config file is invalid.167        """168        try:169            self._config.read(filepath)170            import ast171            self.connection.timeout = \172                self._config.getint("Connection", "timeout")173            self.connection.verify = \174                self._config.getboolean("Connection", "verify")175            self.connection.cert = \176                self._config.get("Connection", "cert")177 178            self.proxies.proxies = \179                ast.literal_eval(self._config.get("Proxies", "proxies"))180            self.proxies.use_env_settings = \181                self._config.getboolean("Proxies", "env_settings")182 183            self.redirect_policy.allow = \184                self._config.getboolean("RedirectPolicy", "allow")185            self.redirect_policy.max_redirects = \186                self._config.getint("RedirectPolicy", "max_redirects")187 188        except (ValueError, EnvironmentError, NoOptionError):189            error = "Supplied config file incompatible."190            raise_with_traceback(ValueError, error)191        finally:192            self._clear_config()193 194 195class ClientRequest(object):196    """Represents a HTTP request.197 198    URL can be given without query parameters, to be added later using "format_parameters".199 200    Instance can be created without data, to be added later using "add_content"201 202    Instance can be created without files, to be added later using "add_formdata"203 204    :param str method: HTTP method (GET, HEAD, etc.)205    :param str url: At least complete scheme/host/path206    :param dict[str,str] headers: HTTP headers207    :param files: Files list.208    :param data: Body to be sent.209    :type data: bytes or str.210    """211    def __init__(self, method, url, headers=None, files=None, data=None):212        # type: (str, str, Mapping[str, str], Any, Any) -> None213        self.method = method214        self.url = url215        self.headers = CaseInsensitiveDict(headers)216        self.files = files217        self.data = data218 219    def __repr__(self):220        return '<ClientRequest [%s]>' % (self.method)221 222    @property223    def body(self):224        """Alias to data."""225        return self.data226 227    @body.setter228    def body(self, value):229        self.data = value230 231    def format_parameters(self, params):232        # type: (Dict[str, str]) -> None233        """Format parameters into a valid query string.234        It's assumed all parameters have already been quoted as235        valid URL strings.236 237        :param dict params: A dictionary of parameters.238        """239        query = urlparse(self.url).query240        if query:241            self.url = self.url.partition('?')[0]242            existing_params = {243                p[0]: p[-1]244                for p in [p.partition('=') for p in query.split('&')]245            }246            params.update(existing_params)247        query_params = ["{}={}".format(k, v) for k, v in params.items()]248        query = '?' + '&'.join(query_params)249        self.url = self.url + query250 251    def add_content(self, data):252        # type: (Optional[Union[Dict[str, Any], ET.Element]]) -> None253        """Add a body to the request.254 255        :param data: Request body data, can be a json serializable256         object (e.g. dictionary) or a generator (e.g. file data).257        """258        if data is None:259            return260 261        if isinstance(data, ET.Element):262            bytes_data = ET.tostring(data, encoding="utf8")263            self.headers['Content-Length'] = str(len(bytes_data))264            self.data = bytes_data265            return266 267        # By default, assume JSON268        try:269            self.data = json.dumps(data)270            self.headers['Content-Length'] = str(len(self.data))271        except TypeError:272            self.data = data273 274    @staticmethod275    def _format_data(data):276        # type: (Union[str, IO]) -> Union[Tuple[None, str], Tuple[Optional[str], IO, str]]277        """Format field data according to whether it is a stream or278        a string for a form-data request.279 280        :param data: The request field data.281        :type data: str or file-like object.282        """283        if hasattr(data, 'read'):284            data = cast(IO, data)285            data_name = None286            try:287                if data.name[0] != '<' and data.name[-1] != '>':288                    data_name = os.path.basename(data.name)289            except (AttributeError, TypeError):290                pass291            return (data_name, data, "application/octet-stream")292        return (None, cast(str, data))293 294    def add_formdata(self, content=None):295        # type: (Optional[Dict[str, str]]) -> None296        """Add data as a multipart form-data request to the request.297 298        We only deal with file-like objects or strings at this point.299        The requests is not yet streamed.300 301        :param dict headers: Any headers to add to the request.302        :param dict content: Dictionary of the fields of the formdata.303        """304        if content is None:305            content = {}306        content_type = self.headers.pop('Content-Type', None) if self.headers else None307 308        if content_type and content_type.lower() == 'application/x-www-form-urlencoded':309            # Do NOT use "add_content" that assumes input is JSON310            self.data = {f: d for f, d in content.items() if d is not None}311        else: # Assume "multipart/form-data"312            self.files = {f: self._format_data(d) for f, d in content.items() if d is not None}313 314class HTTPClientResponse(object):315    """Represent a HTTP response.316 317    No body is defined here on purpose, since async pipeline318    will provide async ways to access the body319 320    You have two different types of body:321    - Full in-memory using "body" as bytes322    """323    def __init__(self, request, internal_response):324        # type: (ClientRequest, Any) -> None325        self.request = request326        self.internal_response = internal_response327        self.status_code = None  # type: Optional[int]328        self.headers = {}  # type: MutableMapping[str, str]329        self.reason = None  # type: Optional[str]330 331    def body(self):332        # type: () -> bytes333        """Return the whole body as bytes in memory.334        """335        pass336 337    def text(self, encoding=None):338        # type: (str) -> str339        """Return the whole body as a string.340 341        :param str encoding: The encoding to apply. If None, use "utf-8-sig".342         Implementation can be smarter if they want (using headers).343        """344        return self.body().decode(encoding or "utf-8-sig")345 346    def raise_for_status(self):347        """Raise for status. Should be overridden, but basic implementation provided.348        """349        if self.status_code >= 400:350            raise ClientRequestError("Received status code {}".format(self.status_code))351 352 353class ClientResponse(HTTPClientResponse):354 355    def stream_download(self, chunk_size=None, callback=None):356        # type: (Optional[int], Optional[Callable]) -> Iterator[bytes]357        """Generator for streaming request body data.358 359        Should be implemented by sub-classes if streaming download360        is supported.361 362        :param callback: Custom callback for monitoring progress.363        :param int chunk_size:364        """365        pass366 367 368class ClientRedirectPolicy(object):369    """Redirect configuration settings.370    """371 372    def __init__(self):373        self.allow = True374        self.max_redirects = 30375 376    def __bool__(self):377        # type: () -> bool378        """Whether redirects are allowed."""379        return self.allow380 381    def __call__(self):382        # type: () -> int383        """Return configuration to be applied to connection."""384        debug = "Configuring redirects: allow=%r, max=%r"385        _LOGGER.debug(debug, self.allow, self.max_redirects)386        return self.max_redirects387 388 389class ClientProxies(object):390    """Proxy configuration settings.391    Proxies can also be configured using HTTP_PROXY and HTTPS_PROXY392    environment variables, in which case set use_env_settings to True.393    """394 395    def __init__(self):396        self.proxies = {}397        self.use_env_settings = True398 399    def __call__(self):400        # type: () -> Dict[str, str]401        """Return configuration to be applied to connection."""402        proxy_string = "\n".join(403            ["    {}: {}".format(k, v) for k, v in self.proxies.items()])404 405        _LOGGER.debug("Configuring proxies: %r", proxy_string)406        debug = "Evaluate proxies against ENV settings: %r"407        _LOGGER.debug(debug, self.use_env_settings)408        return self.proxies409 410    def add(self, protocol, proxy_url):411        # type: (str, str) -> None412        """Add proxy.413 414        :param str protocol: Protocol for which proxy is to be applied. Can415         be 'http', 'https', etc. Can also include host.416        :param str proxy_url: The proxy URL. Where basic auth is required,417         use the format: http://user:password@host418        """419        self.proxies[protocol] = proxy_url420 421 422class ClientConnection(object):423    """Request connection configuration settings.424    """425 426    def __init__(self):427        self.timeout = 100428        self.verify = True429        self.cert = None430        self.data_block_size = 4096431 432    def __call__(self):433        # type: () -> Dict[str, Union[str, int]]434        """Return configuration to be applied to connection."""435        debug = "Configuring request: timeout=%r, verify=%r, cert=%r"436        _LOGGER.debug(debug, self.timeout, self.verify, self.cert)437        return {'timeout': self.timeout,438                'verify': self.verify,439                'cert': self.cert}440 441 442__all__ = [443    'ClientRequest',444    'ClientResponse',445    'HTTPSender',446    # Generic HTTP configuration447    'HTTPSenderConfiguration',448    'ClientRedirectPolicy',449    'ClientProxies',450    'ClientConnection'451]452 453try:454    from .async_abc import AsyncHTTPSender, AsyncClientResponse  # pylint: disable=unused-import455    from .async_abc import __all__ as _async_all456    __all__ += _async_all457except SyntaxError: # Python 2458    pass459except ImportError: # pyinstaller won't include Py3 files in Py2.7 mode460    pass461