codekingpro/portable-devtools

1# Copyright 2017 Google LLC2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15"""Decorators for applying timeout arguments to functions.16 17These decorators are used to wrap API methods to apply either a18Deadline-dependent (recommended), constant (DEPRECATED) or exponential19(DEPRECATED) timeout argument.20 21For example, imagine an API method that can take a while to return results,22such as one that might block until a resource is ready:23 24.. code-block:: python25 26    def is_thing_ready(timeout=None):27        response = requests.get('https://example.com/is_thing_ready')28        response.raise_for_status()29        return response.json()30 31This module allows a function like this to be wrapped so that timeouts are32automatically determined, for example:33 34.. code-block:: python35 36    timeout_ = timeout.ExponentialTimeout()37    is_thing_ready_with_timeout = timeout_(is_thing_ready)38 39    for n in range(10):40        try:41            is_thing_ready_with_timeout({'example': 'data'})42        except:43            pass44 45In this example the first call to ``is_thing_ready`` will have a relatively46small timeout (like 1 second). If the resource is available and the request47completes quickly, the loop exits. But, if the resource isn't yet available48and the request times out, it'll be retried - this time with a larger timeout.49 50In the broader context these decorators are typically combined with51:mod:`google.api_core.retry` to implement API methods with a signature that52matches ``api_method(request, timeout=None, retry=None)``.53"""54 55from __future__ import unicode_literals56 57import datetime58import functools59 60from google.api_core import datetime_helpers61 62_DEFAULT_INITIAL_TIMEOUT = 5.0  # seconds63_DEFAULT_MAXIMUM_TIMEOUT = 30.0  # seconds64_DEFAULT_TIMEOUT_MULTIPLIER = 2.065# If specified, must be in seconds. If none, deadline is not used in the66# timeout calculation.67_DEFAULT_DEADLINE = None68 69 70class TimeToDeadlineTimeout(object):71    """A decorator that decreases timeout set for an RPC based on how much time72    has left till its deadline. The deadline is calculated as73    ``now + initial_timeout`` when this decorator is first called for an rpc.74 75    In other words this decorator implements deadline semantics in terms of a76    sequence of decreasing timeouts t0 > t1 > t2 ... tn >= 0.77 78    Args:79        timeout (Optional[float]): the timeout (in seconds) to applied to the80            wrapped function. If `None`, the target function is expected to81            never timeout.82    """83 84    def __init__(self, timeout=None, clock=datetime_helpers.utcnow):85        self._timeout = timeout86        self._clock = clock87 88    def __call__(self, func):89        """Apply the timeout decorator.90 91        Args:92            func (Callable): The function to apply the timeout argument to.93                This function must accept a timeout keyword argument.94 95        Returns:96            Callable: The wrapped function.97        """98 99        first_attempt_timestamp = self._clock().timestamp()100 101        @functools.wraps(func)102        def func_with_timeout(*args, **kwargs):103            """Wrapped function that adds timeout."""104 105            remaining_timeout = self._timeout106            if remaining_timeout is not None:107                # All calculations are in seconds108                now_timestamp = self._clock().timestamp()109 110                # To avoid usage of nonlocal but still have round timeout111                # numbers for first attempt (in most cases the only attempt made112                # for an RPC.113                if now_timestamp - first_attempt_timestamp < 0.001:114                    now_timestamp = first_attempt_timestamp115 116                time_since_first_attempt = now_timestamp - first_attempt_timestamp117                # Avoid setting negative timeout118                kwargs["timeout"] = max(0, self._timeout - time_since_first_attempt)119 120            return func(*args, **kwargs)121 122        return func_with_timeout123 124    def __str__(self):125        return "<TimeToDeadlineTimeout timeout={:.1f}>".format(self._timeout)126 127 128class ConstantTimeout(object):129    """A decorator that adds a constant timeout argument.130 131    DEPRECATED: use ``TimeToDeadlineTimeout`` instead.132 133    This is effectively equivalent to134    ``functools.partial(func, timeout=timeout)``.135 136    Args:137        timeout (Optional[float]): the timeout (in seconds) to applied to the138            wrapped function. If `None`, the target function is expected to139            never timeout.140    """141 142    def __init__(self, timeout=None):143        self._timeout = timeout144 145    def __call__(self, func):146        """Apply the timeout decorator.147 148        Args:149            func (Callable): The function to apply the timeout argument to.150                This function must accept a timeout keyword argument.151 152        Returns:153            Callable: The wrapped function.154        """155 156        @functools.wraps(func)157        def func_with_timeout(*args, **kwargs):158            """Wrapped function that adds timeout."""159            kwargs["timeout"] = self._timeout160            return func(*args, **kwargs)161 162        return func_with_timeout163 164    def __str__(self):165        return "<ConstantTimeout timeout={:.1f}>".format(self._timeout)166 167 168def _exponential_timeout_generator(initial, maximum, multiplier, deadline):169    """A generator that yields exponential timeout values.170 171    Args:172        initial (float): The initial timeout.173        maximum (float): The maximum timeout.174        multiplier (float): The multiplier applied to the timeout.175        deadline (float): The overall deadline across all invocations.176 177    Yields:178        float: A timeout value.179    """180    if deadline is not None:181        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(182            seconds=deadline183        )184    else:185        deadline_datetime = datetime.datetime.max186 187    timeout = initial188    while True:189        now = datetime_helpers.utcnow()190        yield min(191            # The calculated timeout based on invocations.192            timeout,193            # The set maximum timeout.194            maximum,195            # The remaining time before the deadline is reached.196            float((deadline_datetime - now).seconds),197        )198        timeout = timeout * multiplier199 200 201class ExponentialTimeout(object):202    """A decorator that adds an exponentially increasing timeout argument.203 204    DEPRECATED: the concept of incrementing timeout exponentially has been205    deprecated. Use ``TimeToDeadlineTimeout`` instead.206 207    This is useful if a function is called multiple times. Each time the208    function is called this decorator will calculate a new timeout parameter209    based on the the number of times the function has been called.210 211    For example212 213    .. code-block:: python214 215    Args:216        initial (float): The initial timeout to pass.217        maximum (float): The maximum timeout for any one call.218        multiplier (float): The multiplier applied to the timeout for each219            invocation.220        deadline (Optional[float]): The overall deadline across all221            invocations. This is used to prevent a very large calculated222            timeout from pushing the overall execution time over the deadline.223            This is especially useful in conjunction with224            :mod:`google.api_core.retry`. If ``None``, the timeouts will not225            be adjusted to accommodate an overall deadline.226    """227 228    def __init__(229        self,230        initial=_DEFAULT_INITIAL_TIMEOUT,231        maximum=_DEFAULT_MAXIMUM_TIMEOUT,232        multiplier=_DEFAULT_TIMEOUT_MULTIPLIER,233        deadline=_DEFAULT_DEADLINE,234    ):235        self._initial = initial236        self._maximum = maximum237        self._multiplier = multiplier238        self._deadline = deadline239 240    def with_deadline(self, deadline):241        """Return a copy of this timeout with the given deadline.242 243        Args:244            deadline (float): The overall deadline across all invocations.245 246        Returns:247            ExponentialTimeout: A new instance with the given deadline.248        """249        return ExponentialTimeout(250            initial=self._initial,251            maximum=self._maximum,252            multiplier=self._multiplier,253            deadline=deadline,254        )255 256    def __call__(self, func):257        """Apply the timeout decorator.258 259        Args:260            func (Callable): The function to apply the timeout argument to.261                This function must accept a timeout keyword argument.262 263        Returns:264            Callable: The wrapped function.265        """266        timeouts = _exponential_timeout_generator(267            self._initial, self._maximum, self._multiplier, self._deadline268        )269 270        @functools.wraps(func)271        def func_with_timeout(*args, **kwargs):272            """Wrapped function that adds timeout."""273            kwargs["timeout"] = next(timeouts)274            return func(*args, **kwargs)275 276        return func_with_timeout277 278    def __str__(self):279        return (280            "<ExponentialTimeout initial={:.1f}, maximum={:.1f}, "281            "multiplier={:.1f}, deadline={:.1f}>".format(282                self._initial, self._maximum, self._multiplier, self._deadline283            )284        )285