codekingpro/portable-devtools

1# Copyright 2015 Google LLC2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15"""Iterators for paging through paged API methods.16 17These iterators simplify the process of paging through API responses18where the request takes a page token and the response is a list of results with19a token for the next page. See `list pagination`_ in the Google API Style Guide20for more details.21 22.. _list pagination:23    https://cloud.google.com/apis/design/design_patterns#list_pagination24 25API clients that have methods that follow the list pagination pattern can26return an :class:`.Iterator`. You can use this iterator to get **all** of27the results across all pages::28 29    >>> results_iterator = client.list_resources()30    >>> list(results_iterator)  # Convert to a list (consumes all values).31 32Or you can walk your way through items and call off the search early if33you find what you're looking for (resulting in possibly fewer requests)::34 35    >>> for resource in results_iterator:36    ...     print(resource.name)37    ...     if not resource.is_valid:38    ...         break39 40At any point, you may check the number of items consumed by referencing the41``num_results`` property of the iterator::42 43    >>> for my_item in results_iterator:44    ...     if results_iterator.num_results >= 10:45    ...         break46 47When iterating, not every new item will send a request to the server.48To iterate based on each page of items (where a page corresponds to49a request)::50 51    >>> for page in results_iterator.pages:52    ...     print('=' * 20)53    ...     print('    Page number: {:d}'.format(iterator.page_number))54    ...     print('  Items in page: {:d}'.format(page.num_items))55    ...     print('     First item: {!r}'.format(next(page)))56    ...     print('Items remaining: {:d}'.format(page.remaining))57    ...     print('Next page token: {}'.format(iterator.next_page_token))58    ====================59        Page number: 160      Items in page: 161         First item: <MyItemClass at 0x7f1d3cccf690>62    Items remaining: 063    Next page token: eav1OzQB0OM8rLdGXOEsyQWSG64    ====================65        Page number: 266      Items in page: 1967         First item: <MyItemClass at 0x7f1d3cccffd0>68    Items remaining: 1869    Next page token: None70 71Then, for each page you can get all the resources on that page by iterating72through it or using :func:`list`::73 74    >>> list(page)75    [76        <MyItemClass at 0x7fd64a098ad0>,77        <MyItemClass at 0x7fd64a098ed0>,78        <MyItemClass at 0x7fd64a098e90>,79    ]80"""81 82import abc83 84 85class Page(object):86    """Single page of results in an iterator.87 88    Args:89        parent (google.api_core.page_iterator.Iterator): The iterator that owns90            the current page.91        items (Sequence[Any]): An iterable (that also defines __len__) of items92            from a raw API response.93        item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]):94            Callable to convert an item from the type in the raw API response95            into the native object. Will be called with the iterator and a96            single item.97        raw_page Optional[google.protobuf.message.Message]:98            The raw page response.99    """100 101    def __init__(self, parent, items, item_to_value, raw_page=None):102        self._parent = parent103        self._num_items = len(items)104        self._remaining = self._num_items105        self._item_iter = iter(items)106        self._item_to_value = item_to_value107        self._raw_page = raw_page108 109    @property110    def raw_page(self):111        """google.protobuf.message.Message"""112        return self._raw_page113 114    @property115    def num_items(self):116        """int: Total items in the page."""117        return self._num_items118 119    @property120    def remaining(self):121        """int: Remaining items in the page."""122        return self._remaining123 124    def __iter__(self):125        """The :class:`Page` is an iterator of items."""126        return self127 128    def __next__(self):129        """Get the next value in the page."""130        item = next(self._item_iter)131        result = self._item_to_value(self._parent, item)132        # Since we've successfully got the next value from the133        # iterator, we update the number of remaining.134        self._remaining -= 1135        return result136 137 138def _item_to_value_identity(iterator, item):139    """An item to value transformer that returns the item un-changed."""140    # pylint: disable=unused-argument141    # We are conforming to the interface defined by Iterator.142    return item143 144 145class Iterator(object, metaclass=abc.ABCMeta):146    """A generic class for iterating through API list responses.147 148    Args:149        client(google.cloud.client.Client): The API client.150        item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]):151            Callable to convert an item from the type in the raw API response152            into the native object. Will be called with the iterator and a153            single item.154        page_token (str): A token identifying a page in a result set to start155            fetching results from.156        max_results (int): The maximum number of results to fetch.157    """158 159    def __init__(160        self,161        client,162        item_to_value=_item_to_value_identity,163        page_token=None,164        max_results=None,165    ):166        self._started = False167        self.__active_iterator = None168 169        self.client = client170        """Optional[Any]: The client that created this iterator."""171        self.item_to_value = item_to_value172        """Callable[Iterator, Any]: Callable to convert an item from the type173            in the raw API response into the native object. Will be called with174            the iterator and a175            single item.176        """177        self.max_results = max_results178        """int: The maximum number of results to fetch"""179 180        # The attributes below will change over the life of the iterator.181        self.page_number = 0182        """int: The current page of results."""183        self.next_page_token = page_token184        """str: The token for the next page of results. If this is set before185            the iterator starts, it effectively offsets the iterator to a186            specific starting point."""187        self.num_results = 0188        """int: The total number of results fetched so far."""189 190    @property191    def pages(self):192        """Iterator of pages in the response.193 194        returns:195            types.GeneratorType[google.api_core.page_iterator.Page]: A196                generator of page instances.197 198        raises:199            ValueError: If the iterator has already been started.200        """201        if self._started:202            raise ValueError("Iterator has already started", self)203        self._started = True204        return self._page_iter(increment=True)205 206    def _items_iter(self):207        """Iterator for each item returned."""208        for page in self._page_iter(increment=False):209            for item in page:210                self.num_results += 1211                yield item212 213    def __iter__(self):214        """Iterator for each item returned.215 216        Returns:217            types.GeneratorType[Any]: A generator of items from the API.218 219        Raises:220            ValueError: If the iterator has already been started.221        """222        if self._started:223            raise ValueError("Iterator has already started", self)224        self._started = True225        return self._items_iter()226 227    def __next__(self):228        if self.__active_iterator is None:229            self.__active_iterator = iter(self)230        return next(self.__active_iterator)231 232    def _page_iter(self, increment):233        """Generator of pages of API responses.234 235        Args:236            increment (bool): Flag indicating if the total number of results237                should be incremented on each page. This is useful since a page238                iterator will want to increment by results per page while an239                items iterator will want to increment per item.240 241        Yields:242            Page: each page of items from the API.243        """244        page = self._next_page()245        while page is not None:246            self.page_number += 1247            if increment:248                self.num_results += page.num_items249            yield page250            page = self._next_page()251 252    @abc.abstractmethod253    def _next_page(self):254        """Get the next page in the iterator.255 256        This does nothing and is intended to be over-ridden by subclasses257        to return the next :class:`Page`.258 259        Raises:260            NotImplementedError: Always, this method is abstract.261        """262        raise NotImplementedError263 264 265def _do_nothing_page_start(iterator, page, response):266    """Helper to provide custom behavior after a :class:`Page` is started.267 268    This is a do-nothing stand-in as the default value.269 270    Args:271        iterator (Iterator): An iterator that holds some request info.272        page (Page): The page that was just created.273        response (Any): The API response for a page.274    """275    # pylint: disable=unused-argument276    pass277 278 279class HTTPIterator(Iterator):280    """A generic class for iterating through HTTP/JSON API list responses.281 282    To make an iterator work, you'll need to provide a way to convert a JSON283    item returned from the API into the object of your choice (via284    ``item_to_value``). You also may need to specify a custom ``items_key`` so285    that a given response (containing a page of results) can be parsed into an286    iterable page of the actual objects you want.287 288    Args:289        client (google.cloud.client.Client): The API client.290        api_request (Callable): The function to use to make API requests.291            Generally, this will be292            :meth:`google.cloud._http.JSONConnection.api_request`.293        path (str): The method path to query for the list of items.294        item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]):295            Callable to convert an item from the type in the JSON response into296            a native object. Will be called with the iterator and a single297            item.298        items_key (str): The key in the API response where the list of items299            can be found.300        page_token (str): A token identifying a page in a result set to start301            fetching results from.302        page_size (int): The maximum number of results to fetch per page303        max_results (int): The maximum number of results to fetch304        extra_params (dict): Extra query string parameters for the305            API call.306        page_start (Callable[307            google.api_core.page_iterator.Iterator,308            google.api_core.page_iterator.Page, dict]): Callable to provide309            any special behavior after a new page has been created. Assumed310            signature takes the :class:`.Iterator` that started the page,311            the :class:`.Page` that was started and the dictionary containing312            the page response.313        next_token (str): The name of the field used in the response for page314            tokens.315 316    .. autoattribute:: pages317    """318 319    _DEFAULT_ITEMS_KEY = "items"320    _PAGE_TOKEN = "pageToken"321    _MAX_RESULTS = "maxResults"322    _NEXT_TOKEN = "nextPageToken"323    _RESERVED_PARAMS = frozenset([_PAGE_TOKEN])324    _HTTP_METHOD = "GET"325 326    def __init__(327        self,328        client,329        api_request,330        path,331        item_to_value,332        items_key=_DEFAULT_ITEMS_KEY,333        page_token=None,334        page_size=None,335        max_results=None,336        extra_params=None,337        page_start=_do_nothing_page_start,338        next_token=_NEXT_TOKEN,339    ):340        super(HTTPIterator, self).__init__(341            client, item_to_value, page_token=page_token, max_results=max_results342        )343        self.api_request = api_request344        self.path = path345        self._items_key = items_key346        self.extra_params = extra_params347        self._page_size = page_size348        self._page_start = page_start349        self._next_token = next_token350        # Verify inputs / provide defaults.351        if self.extra_params is None:352            self.extra_params = {}353        self._verify_params()354 355    def _verify_params(self):356        """Verifies the parameters don't use any reserved parameter.357 358        Raises:359            ValueError: If a reserved parameter is used.360        """361        reserved_in_use = self._RESERVED_PARAMS.intersection(self.extra_params)362        if reserved_in_use:363            raise ValueError("Using a reserved parameter", reserved_in_use)364 365    def _next_page(self):366        """Get the next page in the iterator.367 368        Returns:369            Optional[Page]: The next page in the iterator or :data:`None` if370                there are no pages left.371        """372        if self._has_next_page():373            response = self._get_next_page_response()374            items = response.get(self._items_key, ())375            page = Page(self, items, self.item_to_value, raw_page=response)376            self._page_start(self, page, response)377            self.next_page_token = response.get(self._next_token)378            return page379        else:380            return None381 382    def _has_next_page(self):383        """Determines whether or not there are more pages with results.384 385        Returns:386            bool: Whether the iterator has more pages.387        """388        if self.page_number == 0:389            return True390 391        if self.max_results is not None:392            if self.num_results >= self.max_results:393                return False394 395        return self.next_page_token is not None396 397    def _get_query_params(self):398        """Getter for query parameters for the next request.399 400        Returns:401            dict: A dictionary of query parameters.402        """403        result = {}404        if self.next_page_token is not None:405            result[self._PAGE_TOKEN] = self.next_page_token406 407        page_size = None408        if self.max_results is not None:409            page_size = self.max_results - self.num_results410            if self._page_size is not None:411                page_size = min(page_size, self._page_size)412        elif self._page_size is not None:413            page_size = self._page_size414 415        if page_size is not None:416            result[self._MAX_RESULTS] = page_size417 418        result.update(self.extra_params)419        return result420 421    def _get_next_page_response(self):422        """Requests the next page from the path provided.423 424        Returns:425            dict: The parsed JSON response of the next page's contents.426 427        Raises:428            ValueError: If the HTTP method is not ``GET`` or ``POST``.429        """430        params = self._get_query_params()431        if self._HTTP_METHOD == "GET":432            return self.api_request(433                method=self._HTTP_METHOD, path=self.path, query_params=params434            )435        elif self._HTTP_METHOD == "POST":436            return self.api_request(437                method=self._HTTP_METHOD, path=self.path, data=params438            )439        else:440            raise ValueError("Unexpected HTTP method", self._HTTP_METHOD)441 442 443class _GAXIterator(Iterator):444    """A generic class for iterating through Cloud gRPC APIs list responses.445 446    Any:447        client (google.cloud.client.Client): The API client.448        page_iter (google.gax.PageIterator): A GAX page iterator to be wrapped449            to conform to the :class:`Iterator` interface.450        item_to_value (Callable[Iterator, Any]): Callable to convert an item451            from the protobuf response into a native object. Will452            be called with the iterator and a single item.453        max_results (int): The maximum number of results to fetch.454 455    .. autoattribute:: pages456    """457 458    def __init__(self, client, page_iter, item_to_value, max_results=None):459        super(_GAXIterator, self).__init__(460            client,461            item_to_value,462            page_token=page_iter.page_token,463            max_results=max_results,464        )465        self._gax_page_iter = page_iter466 467    def _next_page(self):468        """Get the next page in the iterator.469 470        Wraps the response from the :class:`~google.gax.PageIterator` in a471        :class:`Page` instance and captures some state at each page.472 473        Returns:474            Optional[Page]: The next page in the iterator or :data:`None` if475                  there are no pages left.476        """477        try:478            items = next(self._gax_page_iter)479            page = Page(self, items, self.item_to_value)480            self.next_page_token = self._gax_page_iter.page_token or None481            return page482        except StopIteration:483            return None484 485 486class GRPCIterator(Iterator):487    """A generic class for iterating through gRPC list responses.488 489    .. note:: The class does not take a ``page_token`` argument because it can490        just be specified in the ``request``.491 492    Args:493        client (google.cloud.client.Client): The API client. This unused by494            this class, but kept to satisfy the :class:`Iterator` interface.495        method (Callable[protobuf.Message]): A bound gRPC method that should496            take a single message for the request.497        request (protobuf.Message): The request message.498        items_field (str): The field in the response message that has the499            items for the page.500        item_to_value (Callable[GRPCIterator, Any]): Callable to convert an501            item from the type in the JSON response into a native object. Will502            be called with the iterator and a single item.503        request_token_field (str): The field in the request message used to504            specify the page token.505        response_token_field (str): The field in the response message that has506            the token for the next page.507        max_results (int): The maximum number of results to fetch.508 509    .. autoattribute:: pages510    """511 512    _DEFAULT_REQUEST_TOKEN_FIELD = "page_token"513    _DEFAULT_RESPONSE_TOKEN_FIELD = "next_page_token"514 515    def __init__(516        self,517        client,518        method,519        request,520        items_field,521        item_to_value=_item_to_value_identity,522        request_token_field=_DEFAULT_REQUEST_TOKEN_FIELD,523        response_token_field=_DEFAULT_RESPONSE_TOKEN_FIELD,524        max_results=None,525    ):526        super(GRPCIterator, self).__init__(527            client, item_to_value, max_results=max_results528        )529        self._method = method530        self._request = request531        self._items_field = items_field532        self._request_token_field = request_token_field533        self._response_token_field = response_token_field534 535    def _next_page(self):536        """Get the next page in the iterator.537 538        Returns:539            Page: The next page in the iterator or :data:`None` if540                there are no pages left.541        """542        if not self._has_next_page():543            return None544 545        if self.next_page_token is not None:546            setattr(self._request, self._request_token_field, self.next_page_token)547 548        response = self._method(self._request)549 550        self.next_page_token = getattr(response, self._response_token_field)551        items = getattr(response, self._items_field)552        page = Page(self, items, self.item_to_value, raw_page=response)553 554        return page555 556    def _has_next_page(self):557        """Determines whether or not there are more pages with results.558 559        Returns:560            bool: Whether the iterator has more pages.561        """562        if self.page_number == 0:563            return True564 565        if self.max_results is not None:566            if self.num_results >= self.max_results:567                return False568 569        # Note: intentionally a falsy check instead of a None check. The RPC570        # can return an empty string indicating no more pages.571        return True if self.next_page_token else False572