codekingpro/portable-devtools

1# Copyright 2016 Google LLC2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#      http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15"""Transport adapter for Requests."""16 17from __future__ import absolute_import18 19import functools20import logging21import numbers22import os23import time24 25try:26    import requests27except ImportError as caught_exc:  # pragma: NO COVER28    raise ImportError(29        "The requests library is not installed from please install the requests package to use the requests transport."30    ) from caught_exc31import requests.adapters  # pylint: disable=ungrouped-imports32import requests.exceptions  # pylint: disable=ungrouped-imports33from requests.packages.urllib3.util.ssl_ import (  # type: ignore34    create_urllib3_context,35)  # pylint: disable=ungrouped-imports36 37from google.auth import environment_vars38from google.auth import exceptions39from google.auth import transport40import google.auth.transport._mtls_helper41from google.oauth2 import service_account42 43_LOGGER = logging.getLogger(__name__)44 45_DEFAULT_TIMEOUT = 120  # in seconds46 47 48class _Response(transport.Response):49    """Requests transport response adapter.50 51    Args:52        response (requests.Response): The raw Requests response.53    """54 55    def __init__(self, response):56        self._response = response57 58    @property59    def status(self):60        return self._response.status_code61 62    @property63    def headers(self):64        return self._response.headers65 66    @property67    def data(self):68        return self._response.content69 70 71class TimeoutGuard(object):72    """A context manager raising an error if the suite execution took too long.73 74    Args:75        timeout (Union[None, Union[float, Tuple[float, float]]]):76            The maximum number of seconds a suite can run without the context77            manager raising a timeout exception on exit. If passed as a tuple,78            the smaller of the values is taken as a timeout. If ``None``, a79            timeout error is never raised.80        timeout_error_type (Optional[Exception]):81            The type of the error to raise on timeout. Defaults to82            :class:`requests.exceptions.Timeout`.83    """84 85    def __init__(self, timeout, timeout_error_type=requests.exceptions.Timeout):86        self._timeout = timeout87        self.remaining_timeout = timeout88        self._timeout_error_type = timeout_error_type89 90    def __enter__(self):91        self._start = time.time()92        return self93 94    def __exit__(self, exc_type, exc_value, traceback):95        if exc_value:96            return  # let the error bubble up automatically97 98        if self._timeout is None:99            return  # nothing to do, the timeout was not specified100 101        elapsed = time.time() - self._start102        deadline_hit = False103 104        if isinstance(self._timeout, numbers.Number):105            self.remaining_timeout = self._timeout - elapsed106            deadline_hit = self.remaining_timeout <= 0107        else:108            self.remaining_timeout = tuple(x - elapsed for x in self._timeout)109            deadline_hit = min(self.remaining_timeout) <= 0110 111        if deadline_hit:112            raise self._timeout_error_type()113 114 115class Request(transport.Request):116    """Requests request adapter.117 118    This class is used internally for making requests using various transports119    in a consistent way. If you use :class:`AuthorizedSession` you do not need120    to construct or use this class directly.121 122    This class can be useful if you want to manually refresh a123    :class:`~google.auth.credentials.Credentials` instance::124 125        import google.auth.transport.requests126        import requests127 128        request = google.auth.transport.requests.Request()129 130        credentials.refresh(request)131 132    Args:133        session (requests.Session): An instance :class:`requests.Session` used134            to make HTTP requests. If not specified, a session will be created.135 136    .. automethod:: __call__137    """138 139    def __init__(self, session=None):140        if not session:141            session = requests.Session()142 143        self.session = session144 145    def __del__(self):146        try:147            if hasattr(self, "session") and self.session is not None:148                self.session.close()149        except TypeError:150            # NOTE: For certain Python binary built, the queue.Empty exception151            # might not be considered a normal Python exception causing152            # TypeError.153            pass154 155    def __call__(156        self,157        url,158        method="GET",159        body=None,160        headers=None,161        timeout=_DEFAULT_TIMEOUT,162        **kwargs163    ):164        """Make an HTTP request using requests.165 166        Args:167            url (str): The URI to be requested.168            method (str): The HTTP method to use for the request. Defaults169                to 'GET'.170            body (bytes): The payload or body in HTTP request.171            headers (Mapping[str, str]): Request headers.172            timeout (Optional[int]): The number of seconds to wait for a173                response from the server. If not specified or if None, the174                requests default timeout will be used.175            kwargs: Additional arguments passed through to the underlying176                requests :meth:`~requests.Session.request` method.177 178        Returns:179            google.auth.transport.Response: The HTTP response.180 181        Raises:182            google.auth.exceptions.TransportError: If any exception occurred.183        """184        try:185            _LOGGER.debug("Making request: %s %s", method, url)186            response = self.session.request(187                method, url, data=body, headers=headers, timeout=timeout, **kwargs188            )189            return _Response(response)190        except requests.exceptions.RequestException as caught_exc:191            new_exc = exceptions.TransportError(caught_exc)192            raise new_exc from caught_exc193 194 195class _MutualTlsAdapter(requests.adapters.HTTPAdapter):196    """197    A TransportAdapter that enables mutual TLS.198 199    Args:200        cert (bytes): client certificate in PEM format201        key (bytes): client private key in PEM format202 203    Raises:204        ImportError: if certifi or pyOpenSSL is not installed205        OpenSSL.crypto.Error: if client cert or key is invalid206    """207 208    def __init__(self, cert, key):209        import certifi210        from OpenSSL import crypto211        import urllib3.contrib.pyopenssl  # type: ignore212 213        urllib3.contrib.pyopenssl.inject_into_urllib3()214 215        pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, key)216        x509 = crypto.load_certificate(crypto.FILETYPE_PEM, cert)217 218        ctx_poolmanager = create_urllib3_context()219        ctx_poolmanager.load_verify_locations(cafile=certifi.where())220        ctx_poolmanager._ctx.use_certificate(x509)221        ctx_poolmanager._ctx.use_privatekey(pkey)222        self._ctx_poolmanager = ctx_poolmanager223 224        ctx_proxymanager = create_urllib3_context()225        ctx_proxymanager.load_verify_locations(cafile=certifi.where())226        ctx_proxymanager._ctx.use_certificate(x509)227        ctx_proxymanager._ctx.use_privatekey(pkey)228        self._ctx_proxymanager = ctx_proxymanager229 230        super(_MutualTlsAdapter, self).__init__()231 232    def init_poolmanager(self, *args, **kwargs):233        kwargs["ssl_context"] = self._ctx_poolmanager234        super(_MutualTlsAdapter, self).init_poolmanager(*args, **kwargs)235 236    def proxy_manager_for(self, *args, **kwargs):237        kwargs["ssl_context"] = self._ctx_proxymanager238        return super(_MutualTlsAdapter, self).proxy_manager_for(*args, **kwargs)239 240 241class _MutualTlsOffloadAdapter(requests.adapters.HTTPAdapter):242    """243    A TransportAdapter that enables mutual TLS and offloads the client side244    signing operation to the signing library.245 246    Args:247        enterprise_cert_file_path (str): the path to a enterprise cert JSON248            file. The file should contain the following field:249 250                {251                    "libs": {252                        "signer_library": "...",253                        "offload_library": "..."254                    }255                }256 257    Raises:258        ImportError: if certifi or pyOpenSSL is not installed259        google.auth.exceptions.MutualTLSChannelError: If mutual TLS channel260            creation failed for any reason.261    """262 263    def __init__(self, enterprise_cert_file_path):264        import certifi265        import urllib3.contrib.pyopenssl266 267        from google.auth.transport import _custom_tls_signer268 269        # Call inject_into_urllib3 to activate certificate checking. See the270        # following links for more info:271        # (1) doc: https://github.com/urllib3/urllib3/blob/cb9ebf8aac5d75f64c8551820d760b72b619beff/src/urllib3/contrib/pyopenssl.py#L31-L32272        # (2) mTLS example: https://github.com/urllib3/urllib3/issues/474#issuecomment-253168415273        urllib3.contrib.pyopenssl.inject_into_urllib3()274 275        self.signer = _custom_tls_signer.CustomTlsSigner(enterprise_cert_file_path)276        self.signer.load_libraries()277 278        poolmanager = create_urllib3_context()279        poolmanager.load_verify_locations(cafile=certifi.where())280        self.signer.attach_to_ssl_context(poolmanager)281        self._ctx_poolmanager = poolmanager282 283        proxymanager = create_urllib3_context()284        proxymanager.load_verify_locations(cafile=certifi.where())285        self.signer.attach_to_ssl_context(proxymanager)286        self._ctx_proxymanager = proxymanager287 288        super(_MutualTlsOffloadAdapter, self).__init__()289 290    def init_poolmanager(self, *args, **kwargs):291        kwargs["ssl_context"] = self._ctx_poolmanager292        super(_MutualTlsOffloadAdapter, self).init_poolmanager(*args, **kwargs)293 294    def proxy_manager_for(self, *args, **kwargs):295        kwargs["ssl_context"] = self._ctx_proxymanager296        return super(_MutualTlsOffloadAdapter, self).proxy_manager_for(*args, **kwargs)297 298 299class AuthorizedSession(requests.Session):300    """A Requests Session class with credentials.301 302    This class is used to perform requests to API endpoints that require303    authorization::304 305        from google.auth.transport.requests import AuthorizedSession306 307        authed_session = AuthorizedSession(credentials)308 309        response = authed_session.request(310            'GET', 'https://www.googleapis.com/storage/v1/b')311 312 313    The underlying :meth:`request` implementation handles adding the314    credentials' headers to the request and refreshing credentials as needed.315 316    This class also supports mutual TLS via :meth:`configure_mtls_channel`317    method. In order to use this method, the `GOOGLE_API_USE_CLIENT_CERTIFICATE`318    environment variable must be explicitly set to ``true``, otherwise it does319    nothing. Assume the environment is set to ``true``, the method behaves in the320    following manner:321 322    If client_cert_callback is provided, client certificate and private323    key are loaded using the callback; if client_cert_callback is None,324    application default SSL credentials will be used. Exceptions are raised if325    there are problems with the certificate, private key, or the loading process,326    so it should be called within a try/except block.327 328    First we set the environment variable to ``true``, then create an :class:`AuthorizedSession`329    instance and specify the endpoints::330 331        regular_endpoint = 'https://pubsub.googleapis.com/v1/projects/{my_project_id}/topics'332        mtls_endpoint = 'https://pubsub.mtls.googleapis.com/v1/projects/{my_project_id}/topics'333 334        authed_session = AuthorizedSession(credentials)335 336    Now we can pass a callback to :meth:`configure_mtls_channel`::337 338        def my_cert_callback():339            # some code to load client cert bytes and private key bytes, both in340            # PEM format.341            some_code_to_load_client_cert_and_key()342            if loaded:343                return cert, key344            raise MyClientCertFailureException()345 346        # Always call configure_mtls_channel within a try/except block.347        try:348            authed_session.configure_mtls_channel(my_cert_callback)349        except:350            # handle exceptions.351 352        if authed_session.is_mtls:353            response = authed_session.request('GET', mtls_endpoint)354        else:355            response = authed_session.request('GET', regular_endpoint)356 357 358    You can alternatively use application default SSL credentials like this::359 360        try:361            authed_session.configure_mtls_channel()362        except:363            # handle exceptions.364 365    Args:366        credentials (google.auth.credentials.Credentials): The credentials to367            add to the request.368        refresh_status_codes (Sequence[int]): Which HTTP status codes indicate369            that credentials should be refreshed and the request should be370            retried.371        max_refresh_attempts (int): The maximum number of times to attempt to372            refresh the credentials and retry the request.373        refresh_timeout (Optional[int]): The timeout value in seconds for374            credential refresh HTTP requests.375        auth_request (google.auth.transport.requests.Request):376            (Optional) An instance of377            :class:`~google.auth.transport.requests.Request` used when378            refreshing credentials. If not passed,379            an instance of :class:`~google.auth.transport.requests.Request`380            is created.381        default_host (Optional[str]): A host like "pubsub.googleapis.com".382            This is used when a self-signed JWT is created from service383            account credentials.384    """385 386    def __init__(387        self,388        credentials,389        refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,390        max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,391        refresh_timeout=None,392        auth_request=None,393        default_host=None,394    ):395        super(AuthorizedSession, self).__init__()396        self.credentials = credentials397        self._refresh_status_codes = refresh_status_codes398        self._max_refresh_attempts = max_refresh_attempts399        self._refresh_timeout = refresh_timeout400        self._is_mtls = False401        self._default_host = default_host402 403        if auth_request is None:404            self._auth_request_session = requests.Session()405 406            # Using an adapter to make HTTP requests robust to network errors.407            # This adapter retrys HTTP requests when network errors occur408            # and the requests seems safely retryable.409            retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)410            self._auth_request_session.mount("https://", retry_adapter)411 412            # Do not pass `self` as the session here, as it can lead to413            # infinite recursion.414            auth_request = Request(self._auth_request_session)415        else:416            self._auth_request_session = None417 418        # Request instance used by internal methods (for example,419        # credentials.refresh).420        self._auth_request = auth_request421 422        # https://google.aip.dev/auth/4111423        # Attempt to use self-signed JWTs when a service account is used.424        if isinstance(self.credentials, service_account.Credentials):425            self.credentials._create_self_signed_jwt(426                "https://{}/".format(self._default_host) if self._default_host else None427            )428 429    def configure_mtls_channel(self, client_cert_callback=None):430        """Configure the client certificate and key for SSL connection.431 432        The function does nothing unless `GOOGLE_API_USE_CLIENT_CERTIFICATE` is433        explicitly set to `true`. In this case if client certificate and key are434        successfully obtained (from the given client_cert_callback or from application435        default SSL credentials), a :class:`_MutualTlsAdapter` instance will be mounted436        to "https://" prefix.437 438        Args:439            client_cert_callback (Optional[Callable[[], (bytes, bytes)]]):440                The optional callback returns the client certificate and private441                key bytes both in PEM format.442                If the callback is None, application default SSL credentials443                will be used.444 445        Raises:446            google.auth.exceptions.MutualTLSChannelError: If mutual TLS channel447                creation failed for any reason.448        """449        use_client_cert = os.getenv(450            environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"451        )452        if use_client_cert != "true":453            self._is_mtls = False454            return455 456        try:457            import OpenSSL458        except ImportError as caught_exc:459            new_exc = exceptions.MutualTLSChannelError(caught_exc)460            raise new_exc from caught_exc461 462        try:463            (464                self._is_mtls,465                cert,466                key,467            ) = google.auth.transport._mtls_helper.get_client_cert_and_key(468                client_cert_callback469            )470 471            if self._is_mtls:472                mtls_adapter = _MutualTlsAdapter(cert, key)473                self.mount("https://", mtls_adapter)474        except (475            exceptions.ClientCertError,476            ImportError,477            OpenSSL.crypto.Error,478        ) as caught_exc:479            new_exc = exceptions.MutualTLSChannelError(caught_exc)480            raise new_exc from caught_exc481 482    def request(483        self,484        method,485        url,486        data=None,487        headers=None,488        max_allowed_time=None,489        timeout=_DEFAULT_TIMEOUT,490        **kwargs491    ):492        """Implementation of Requests' request.493 494        Args:495            timeout (Optional[Union[float, Tuple[float, float]]]):496                The amount of time in seconds to wait for the server response497                with each individual request. Can also be passed as a tuple498                ``(connect_timeout, read_timeout)``. See :meth:`requests.Session.request`499                documentation for details.500            max_allowed_time (Optional[float]):501                If the method runs longer than this, a ``Timeout`` exception is502                automatically raised. Unlike the ``timeout`` parameter, this503                value applies to the total method execution time, even if504                multiple requests are made under the hood.505 506                Mind that it is not guaranteed that the timeout error is raised507                at ``max_allowed_time``. It might take longer, for example, if508                an underlying request takes a lot of time, but the request509                itself does not timeout, e.g. if a large file is being510                transmitted. The timout error will be raised after such511                request completes.512        """513        # pylint: disable=arguments-differ514        # Requests has a ton of arguments to request, but only two515        # (method, url) are required. We pass through all of the other516        # arguments to super, so no need to exhaustively list them here.517 518        # Use a kwarg for this instead of an attribute to maintain519        # thread-safety.520        _credential_refresh_attempt = kwargs.pop("_credential_refresh_attempt", 0)521 522        # Make a copy of the headers. They will be modified by the credentials523        # and we want to pass the original headers if we recurse.524        request_headers = headers.copy() if headers is not None else {}525 526        # Do not apply the timeout unconditionally in order to not override the527        # _auth_request's default timeout.528        auth_request = (529            self._auth_request530            if timeout is None531            else functools.partial(self._auth_request, timeout=timeout)532        )533 534        remaining_time = max_allowed_time535 536        with TimeoutGuard(remaining_time) as guard:537            self.credentials.before_request(auth_request, method, url, request_headers)538        remaining_time = guard.remaining_timeout539 540        with TimeoutGuard(remaining_time) as guard:541            response = super(AuthorizedSession, self).request(542                method,543                url,544                data=data,545                headers=request_headers,546                timeout=timeout,547                **kwargs548            )549        remaining_time = guard.remaining_timeout550 551        # If the response indicated that the credentials needed to be552        # refreshed, then refresh the credentials and re-attempt the553        # request.554        # A stored token may expire between the time it is retrieved and555        # the time the request is made, so we may need to try twice.556        if (557            response.status_code in self._refresh_status_codes558            and _credential_refresh_attempt < self._max_refresh_attempts559        ):560 561            _LOGGER.info(562                "Refreshing credentials due to a %s response. Attempt %s/%s.",563                response.status_code,564                _credential_refresh_attempt + 1,565                self._max_refresh_attempts,566            )567 568            # Do not apply the timeout unconditionally in order to not override the569            # _auth_request's default timeout.570            auth_request = (571                self._auth_request572                if timeout is None573                else functools.partial(self._auth_request, timeout=timeout)574            )575 576            with TimeoutGuard(remaining_time) as guard:577                self.credentials.refresh(auth_request)578            remaining_time = guard.remaining_timeout579 580            # Recurse. Pass in the original headers, not our modified set, but581            # do pass the adjusted max allowed time (i.e. the remaining total time).582            return self.request(583                method,584                url,585                data=data,586                headers=headers,587                max_allowed_time=remaining_time,588                timeout=timeout,589                _credential_refresh_attempt=_credential_refresh_attempt + 1,590                **kwargs591            )592 593        return response594 595    @property596    def is_mtls(self):597        """Indicates if the created SSL channel is mutual TLS."""598        return self._is_mtls599 600    def close(self):601        if self._auth_request_session is not None:602            self._auth_request_session.close()603        super(AuthorizedSession, self).close()604