codekingpro/portable-devtools

1# Copyright 2017 Google LLC2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15"""Helpers for retrying functions with exponential back-off.16 17The :class:`Retry` decorator can be used to retry functions that raise18exceptions using exponential backoff. Because a exponential sleep algorithm is19used, the retry is limited by a `timeout`. The timeout determines the window20in which retries will be attempted. This is used instead of total number of retries21because it is difficult to ascertain the amount of time a function can block22when using total number of retries and exponential backoff.23 24By default, this decorator will retry transient25API errors (see :func:`if_transient_error`). For example:26 27.. code-block:: python28 29    @retry.Retry()30    def call_flaky_rpc():31        return client.flaky_rpc()32 33    # Will retry flaky_rpc() if it raises transient API errors.34    result = call_flaky_rpc()35 36You can pass a custom predicate to retry on different exceptions, such as37waiting for an eventually consistent item to be available:38 39.. code-block:: python40 41    @retry.Retry(predicate=if_exception_type(exceptions.NotFound))42    def check_if_exists():43        return client.does_thing_exist()44 45    is_available = check_if_exists()46 47Some client library methods apply retry automatically. These methods can accept48a ``retry`` parameter that allows you to configure the behavior:49 50.. code-block:: python51 52    my_retry = retry.Retry(timeout=60)53    result = client.some_method(retry=my_retry)54 55"""56 57from __future__ import annotations58 59import functools60import sys61import time62import inspect63import warnings64from typing import Any, Callable, Iterable, TypeVar, TYPE_CHECKING65 66from google.api_core.retry.retry_base import _BaseRetry67from google.api_core.retry.retry_base import _retry_error_helper68from google.api_core.retry.retry_base import exponential_sleep_generator69from google.api_core.retry.retry_base import build_retry_error70from google.api_core.retry.retry_base import RetryFailureReason71 72 73if TYPE_CHECKING:74    if sys.version_info >= (3, 10):75        from typing import ParamSpec76    else:77        from typing_extensions import ParamSpec78 79    _P = ParamSpec("_P")  # target function call parameters80    _R = TypeVar("_R")  # target function returned value81 82_ASYNC_RETRY_WARNING = "Using the synchronous google.api_core.retry.Retry with asynchronous calls may lead to unexpected results. Please use google.api_core.retry_async.AsyncRetry instead."83 84 85def retry_target(86    target: Callable[_P, _R],87    predicate: Callable[[Exception], bool],88    sleep_generator: Iterable[float],89    timeout: float | None = None,90    on_error: Callable[[Exception], None] | None = None,91    exception_factory: Callable[92        [list[Exception], RetryFailureReason, float | None],93        tuple[Exception, Exception | None],94    ] = build_retry_error,95    **kwargs,96):97    """Call a function and retry if it fails.98 99    This is the lowest-level retry helper. Generally, you'll use the100    higher-level retry helper :class:`Retry`.101 102    Args:103        target(Callable): The function to call and retry. This must be a104            nullary function - apply arguments with `functools.partial`.105        predicate (Callable[Exception]): A callable used to determine if an106            exception raised by the target should be considered retryable.107            It should return True to retry or False otherwise.108        sleep_generator (Iterable[float]): An infinite iterator that determines109            how long to sleep between retries.110        timeout (Optional[float]): How long to keep retrying the target.111            Note: timeout is only checked before initiating a retry, so the target may112            run past the timeout value as long as it is healthy.113        on_error (Optional[Callable[Exception]]): If given, the on_error114            callback will be called with each retryable exception raised by the115            target. Any error raised by this function will *not* be caught.116        exception_factory: A function that is called when the retryable reaches117            a terminal failure state, used to construct an exception to be raised.118            It takes a list of all exceptions encountered, a retry.RetryFailureReason119            enum indicating the failure cause, and the original timeout value120            as arguments. It should return a tuple of the exception to be raised,121            along with the cause exception if any. The default implementation will raise122            a RetryError on timeout, or the last exception encountered otherwise.123        deadline (float): DEPRECATED: use ``timeout`` instead. For backward124            compatibility, if specified it will override ``timeout`` parameter.125 126    Returns:127        Any: the return value of the target function.128 129    Raises:130        ValueError: If the sleep generator stops yielding values.131        Exception: a custom exception specified by the exception_factory if provided.132            If no exception_factory is provided:133                google.api_core.RetryError: If the timeout is exceeded while retrying.134                Exception: If the target raises an error that isn't retryable.135    """136 137    timeout = kwargs.get("deadline", timeout)138 139    deadline = time.monotonic() + timeout if timeout is not None else None140    error_list: list[Exception] = []141 142    for sleep in sleep_generator:143        try:144            result = target()145            if inspect.isawaitable(result):146                warnings.warn(_ASYNC_RETRY_WARNING)147            return result148 149        # pylint: disable=broad-except150        # This function explicitly must deal with broad exceptions.151        except Exception as exc:152            # defer to shared logic for handling errors153            _retry_error_helper(154                exc,155                deadline,156                sleep,157                error_list,158                predicate,159                on_error,160                exception_factory,161                timeout,162            )163            # if exception not raised, sleep before next attempt164            time.sleep(sleep)165 166    raise ValueError("Sleep generator stopped yielding sleep values.")167 168 169class Retry(_BaseRetry):170    """Exponential retry decorator for unary synchronous RPCs.171 172    This class is a decorator used to add retry or polling behavior to an RPC173    call.174 175    Although the default behavior is to retry transient API errors, a176    different predicate can be provided to retry other exceptions.177 178    There are two important concepts that retry/polling behavior may operate on,179    Deadline and Timeout, which need to be properly defined for the correct180    usage of this class and the rest of the library.181 182    Deadline: a fixed point in time by which a certain operation must183    terminate. For example, if a certain operation has a deadline184    "2022-10-18T23:30:52.123Z" it must terminate (successfully or with an185    error) by that time, regardless of when it was started or whether it186    was started at all.187 188    Timeout: the maximum duration of time after which a certain operation189    must terminate (successfully or with an error). The countdown begins right190    after an operation was started. For example, if an operation was started at191    09:24:00 with timeout of 75 seconds, it must terminate no later than192    09:25:15.193 194    Unfortunately, in the past this class (and the api-core library as a whole) has not195    been properly distinguishing the concepts of "timeout" and "deadline", and the196    ``deadline`` parameter has meant ``timeout``. That is why197    ``deadline`` has been deprecated and ``timeout`` should be used instead. If the198    ``deadline`` parameter is set, it will override the ``timeout`` parameter.199    In other words, ``retry.deadline`` should be treated as just a deprecated alias for200    ``retry.timeout``.201 202    Said another way, it is safe to assume that this class and the rest of this203    library operate in terms of timeouts (not deadlines) unless explicitly204    noted the usage of deadline semantics.205 206    It is also important to207    understand the three most common applications of the Timeout concept in the208    context of this library.209 210    Usually the generic Timeout term may stand for one of the following actual211    timeouts: RPC Timeout, Retry Timeout, or Polling Timeout.212 213    RPC Timeout: a value supplied by the client to the server so214    that the server side knows the maximum amount of time it is expected to215    spend handling that specific RPC. For example, in the case of gRPC transport,216    RPC Timeout is represented by setting "grpc-timeout" header in the HTTP2217    request. The `timeout` property of this class normally never represents the218    RPC Timeout as it is handled separately by the ``google.api_core.timeout``219    module of this library.220 221    Retry Timeout: this is the most common meaning of the ``timeout`` property222    of this class, and defines how long a certain RPC may be retried in case223    the server returns an error.224 225    Polling Timeout: defines how long the226    client side is allowed to call the polling RPC repeatedly to check a status of a227    long-running operation. Each polling RPC is228    expected to succeed (its errors are supposed to be handled by the retry229    logic). The decision as to whether a new polling attempt needs to be made is based230    not on the RPC status code but  on the status of the returned231    status of an operation. In other words: we will poll a long-running operation until232    the operation is done or the polling timeout expires. Each poll will inform us of233    the status of the operation. The poll consists of an RPC to the server that may234    itself be retried as per the poll-specific retry settings in case of errors. The235    operation-level retry settings do NOT apply to polling-RPC retries.236 237    With the actual timeout types being defined above, the client libraries238    often refer to just Timeout without clarifying which type specifically239    that is. In that case the actual timeout type (sometimes also referred to as240    Logical Timeout) can be determined from the context. If it is a unary rpc241    call (i.e. a regular one) Timeout usually stands for the RPC Timeout (if242    provided directly as a standalone value) or Retry Timeout (if provided as243    ``retry.timeout`` property of the unary RPC's retry config). For244    ``Operation`` or ``PollingFuture`` in general Timeout stands for245    Polling Timeout.246 247    Args:248        predicate (Callable[Exception]): A callable that should return ``True``249            if the given exception is retryable.250        initial (float): The minimum amount of time to delay in seconds. This251            must be greater than 0.252        maximum (float): The maximum amount of time to delay in seconds.253        multiplier (float): The multiplier applied to the delay.254        timeout (Optional[float]): How long to keep retrying, in seconds.255            Note: timeout is only checked before initiating a retry, so the target may256            run past the timeout value as long as it is healthy.257        on_error (Callable[Exception]): A function to call while processing258            a retryable exception. Any error raised by this function will259            *not* be caught.260        deadline (float): DEPRECATED: use `timeout` instead. For backward261            compatibility, if specified it will override the ``timeout`` parameter.262    """263 264    def __call__(265        self,266        func: Callable[_P, _R],267        on_error: Callable[[Exception], Any] | None = None,268    ) -> Callable[_P, _R]:269        """Wrap a callable with retry behavior.270 271        Args:272            func (Callable): The callable to add retry behavior to.273            on_error (Optional[Callable[Exception]]): If given, the274                on_error callback will be called with each retryable exception275                raised by the wrapped function. Any error raised by this276                function will *not* be caught. If on_error was specified in the277                constructor, this value will be ignored.278 279        Returns:280            Callable: A callable that will invoke ``func`` with retry281                behavior.282        """283        if self._on_error is not None:284            on_error = self._on_error285 286        @functools.wraps(func)287        def retry_wrapped_func(*args: _P.args, **kwargs: _P.kwargs) -> _R:288            """A wrapper that calls target function with retry."""289            target = functools.partial(func, *args, **kwargs)290            sleep_generator = exponential_sleep_generator(291                self._initial, self._maximum, multiplier=self._multiplier292            )293            return retry_target(294                target,295                self._predicate,296                sleep_generator,297                timeout=self._timeout,298                on_error=on_error,299            )300 301        return retry_wrapped_func302