codekingpro/portable-devtools

1# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License"). You4# may not use this file except in compliance with the License. A copy of5# the License is located at6#7# https://aws.amazon.com/apache2.0/8#9# or in the "license" file accompanying this file. This file is10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF11# ANY KIND, either express or implied. See the License for the specific12# language governing permissions and limitations under the License.13 14import copy15import logging16 17from botocore import xform_name18from botocore.utils import merge_dicts19 20from ..docs import docstring21from .action import BatchAction22from .params import create_request_parameters23from .response import ResourceHandler24 25logger = logging.getLogger(__name__)26 27 28class ResourceCollection:29    """30    Represents a collection of resources, which can be iterated through,31    optionally with filtering. Collections automatically handle pagination32    for you.33 34    See :ref:`guide_collections` for a high-level overview of collections,35    including when remote service requests are performed.36 37    :type model: :py:class:`~boto3.resources.model.Collection`38    :param model: Collection model39    :type parent: :py:class:`~boto3.resources.base.ServiceResource`40    :param parent: The collection's parent resource41    :type handler: :py:class:`~boto3.resources.response.ResourceHandler`42    :param handler: The resource response handler used to create resource43                    instances44    """45 46    def __init__(self, model, parent, handler, **kwargs):47        self._model = model48        self._parent = parent49        self._py_operation_name = xform_name(model.request.operation)50        self._handler = handler51        self._params = copy.deepcopy(kwargs)52 53    def __repr__(self):54        return '{}({}, {})'.format(55            self.__class__.__name__,56            self._parent,57            '{}.{}'.format(58                self._parent.meta.service_name, self._model.resource.type59            ),60        )61 62    def __iter__(self):63        """64        A generator which yields resource instances after doing the65        appropriate service operation calls and handling any pagination66        on your behalf.67 68        Page size, item limit, and filter parameters are applied69        if they have previously been set.70 71            >>> bucket = s3.Bucket('boto3')72            >>> for obj in bucket.objects.all():73            ...     print(obj.key)74            'key1'75            'key2'76 77        """78        limit = self._params.get('limit', None)79 80        count = 081        for page in self.pages():82            for item in page:83                yield item84 85                # If the limit is set and has been reached, then86                # we stop processing items here.87                count += 188                if limit is not None and count >= limit:89                    return90 91    def _clone(self, **kwargs):92        """93        Create a clone of this collection. This is used by the methods94        below to provide a chainable interface that returns copies95        rather than the original. This allows things like:96 97            >>> base = collection.filter(Param1=1)98            >>> query1 = base.filter(Param2=2)99            >>> query2 = base.filter(Param3=3)100            >>> query1.params101            {'Param1': 1, 'Param2': 2}102            >>> query2.params103            {'Param1': 1, 'Param3': 3}104 105        :rtype: :py:class:`ResourceCollection`106        :return: A clone of this resource collection107        """108        params = copy.deepcopy(self._params)109        merge_dicts(params, kwargs, append_lists=True)110        clone = self.__class__(111            self._model, self._parent, self._handler, **params112        )113        return clone114 115    def pages(self):116        """117        A generator which yields pages of resource instances after118        doing the appropriate service operation calls and handling119        any pagination on your behalf. Non-paginated calls will120        return a single page of items.121 122        Page size, item limit, and filter parameters are applied123        if they have previously been set.124 125            >>> bucket = s3.Bucket('boto3')126            >>> for page in bucket.objects.pages():127            ...     for obj in page:128            ...         print(obj.key)129            'key1'130            'key2'131 132        :rtype: list(:py:class:`~boto3.resources.base.ServiceResource`)133        :return: List of resource instances134        """135        client = self._parent.meta.client136        cleaned_params = self._params.copy()137        limit = cleaned_params.pop('limit', None)138        page_size = cleaned_params.pop('page_size', None)139        params = create_request_parameters(self._parent, self._model.request)140        merge_dicts(params, cleaned_params, append_lists=True)141 142        # Is this a paginated operation? If so, we need to get an143        # iterator for the various pages. If not, then we simply144        # call the operation and return the result as a single145        # page in a list. For non-paginated results, we just ignore146        # the page size parameter.147        if client.can_paginate(self._py_operation_name):148            logger.debug(149                'Calling paginated %s:%s with %r',150                self._parent.meta.service_name,151                self._py_operation_name,152                params,153            )154            paginator = client.get_paginator(self._py_operation_name)155            pages = paginator.paginate(156                PaginationConfig={'MaxItems': limit, 'PageSize': page_size},157                **params158            )159        else:160            logger.debug(161                'Calling %s:%s with %r',162                self._parent.meta.service_name,163                self._py_operation_name,164                params,165            )166            pages = [getattr(client, self._py_operation_name)(**params)]167 168        # Now that we have a page iterator or single page of results169        # we start processing and yielding individual items.170        count = 0171        for page in pages:172            page_items = []173            for item in self._handler(self._parent, params, page):174                page_items.append(item)175 176                # If the limit is set and has been reached, then177                # we stop processing items here.178                count += 1179                if limit is not None and count >= limit:180                    break181 182            yield page_items183 184            # Stop reading pages if we've reached out limit185            if limit is not None and count >= limit:186                break187 188    def all(self):189        """190        Get all items from the collection, optionally with a custom191        page size and item count limit.192 193        This method returns an iterable generator which yields194        individual resource instances. Example use::195 196            # Iterate through items197            >>> for queue in sqs.queues.all():198            ...     print(queue.url)199            'https://url1'200            'https://url2'201 202            # Convert to list203            >>> queues = list(sqs.queues.all())204            >>> len(queues)205            2206        """207        return self._clone()208 209    def filter(self, **kwargs):210        """211        Get items from the collection, passing keyword arguments along212        as parameters to the underlying service operation, which are213        typically used to filter the results.214 215        This method returns an iterable generator which yields216        individual resource instances. Example use::217 218            # Iterate through items219            >>> for queue in sqs.queues.filter(Param='foo'):220            ...     print(queue.url)221            'https://url1'222            'https://url2'223 224            # Convert to list225            >>> queues = list(sqs.queues.filter(Param='foo'))226            >>> len(queues)227            2228 229        :rtype: :py:class:`ResourceCollection`230        """231        return self._clone(**kwargs)232 233    def limit(self, count):234        """235        Return at most this many resources.236 237            >>> for bucket in s3.buckets.limit(5):238            ...     print(bucket.name)239            'bucket1'240            'bucket2'241            'bucket3'242            'bucket4'243            'bucket5'244 245        :type count: int246        :param count: Return no more than this many items247        :rtype: :py:class:`ResourceCollection`248        """249        return self._clone(limit=count)250 251    def page_size(self, count):252        """253        Fetch at most this many resources per service request.254 255            >>> for obj in s3.Bucket('boto3').objects.page_size(100):256            ...     print(obj.key)257 258        :type count: int259        :param count: Fetch this many items per request260        :rtype: :py:class:`ResourceCollection`261        """262        return self._clone(page_size=count)263 264 265class CollectionManager:266    """267    A collection manager provides access to resource collection instances,268    which can be iterated and filtered. The manager exposes some269    convenience functions that are also found on resource collections,270    such as :py:meth:`~ResourceCollection.all` and271    :py:meth:`~ResourceCollection.filter`.272 273    Get all items::274 275        >>> for bucket in s3.buckets.all():276        ...     print(bucket.name)277 278    Get only some items via filtering::279 280        >>> for queue in sqs.queues.filter(QueueNamePrefix='AWS'):281        ...     print(queue.url)282 283    Get whole pages of items:284 285        >>> for page in s3.Bucket('boto3').objects.pages():286        ...     for obj in page:287        ...         print(obj.key)288 289    A collection manager is not iterable. You **must** call one of the290    methods that return a :py:class:`ResourceCollection` before trying291    to iterate, slice, or convert to a list.292 293    See the :ref:`guide_collections` guide for a high-level overview294    of collections, including when remote service requests are performed.295 296    :type collection_model: :py:class:`~boto3.resources.model.Collection`297    :param model: Collection model298 299    :type parent: :py:class:`~boto3.resources.base.ServiceResource`300    :param parent: The collection's parent resource301 302    :type factory: :py:class:`~boto3.resources.factory.ResourceFactory`303    :param factory: The resource factory to create new resources304 305    :type service_context: :py:class:`~boto3.utils.ServiceContext`306    :param service_context: Context about the AWS service307    """308 309    # The class to use when creating an iterator310    _collection_cls = ResourceCollection311 312    def __init__(self, collection_model, parent, factory, service_context):313        self._model = collection_model314        operation_name = self._model.request.operation315        self._parent = parent316 317        search_path = collection_model.resource.path318        self._handler = ResourceHandler(319            search_path=search_path,320            factory=factory,321            resource_model=collection_model.resource,322            service_context=service_context,323            operation_name=operation_name,324        )325 326    def __repr__(self):327        return '{}({}, {})'.format(328            self.__class__.__name__,329            self._parent,330            '{}.{}'.format(331                self._parent.meta.service_name, self._model.resource.type332            ),333        )334 335    def iterator(self, **kwargs):336        """337        Get a resource collection iterator from this manager.338 339        :rtype: :py:class:`ResourceCollection`340        :return: An iterable representing the collection of resources341        """342        return self._collection_cls(343            self._model, self._parent, self._handler, **kwargs344        )345 346    # Set up some methods to proxy ResourceCollection methods347    def all(self):348        return self.iterator()349 350    all.__doc__ = ResourceCollection.all.__doc__351 352    def filter(self, **kwargs):353        return self.iterator(**kwargs)354 355    filter.__doc__ = ResourceCollection.filter.__doc__356 357    def limit(self, count):358        return self.iterator(limit=count)359 360    limit.__doc__ = ResourceCollection.limit.__doc__361 362    def page_size(self, count):363        return self.iterator(page_size=count)364 365    page_size.__doc__ = ResourceCollection.page_size.__doc__366 367    def pages(self):368        return self.iterator().pages()369 370    pages.__doc__ = ResourceCollection.pages.__doc__371 372 373class CollectionFactory:374    """375    A factory to create new376    :py:class:`CollectionManager` and :py:class:`ResourceCollection`377    subclasses from a :py:class:`~boto3.resources.model.Collection`378    model. These subclasses include methods to perform batch operations.379    """380 381    def load_from_definition(382        self, resource_name, collection_model, service_context, event_emitter383    ):384        """385        Loads a collection from a model, creating a new386        :py:class:`CollectionManager` subclass387        with the correct properties and methods, named based on the service388        and resource name, e.g. ec2.InstanceCollectionManager. It also389        creates a new :py:class:`ResourceCollection` subclass which is used390        by the new manager class.391 392        :type resource_name: string393        :param resource_name: Name of the resource to look up. For services,394                              this should match the ``service_name``.395 396        :type service_context: :py:class:`~boto3.utils.ServiceContext`397        :param service_context: Context about the AWS service398 399        :type event_emitter: :py:class:`~botocore.hooks.HierarchialEmitter`400        :param event_emitter: An event emitter401 402        :rtype: Subclass of :py:class:`CollectionManager`403        :return: The collection class.404        """405        attrs = {}406        collection_name = collection_model.name407 408        # Create the batch actions for a collection409        self._load_batch_actions(410            attrs,411            resource_name,412            collection_model,413            service_context.service_model,414            event_emitter,415        )416        # Add the documentation to the collection class's methods417        self._load_documented_collection_methods(418            attrs=attrs,419            resource_name=resource_name,420            collection_model=collection_model,421            service_model=service_context.service_model,422            event_emitter=event_emitter,423            base_class=ResourceCollection,424        )425 426        if service_context.service_name == resource_name:427            cls_name = '{}.{}Collection'.format(428                service_context.service_name, collection_name429            )430        else:431            cls_name = '{}.{}.{}Collection'.format(432                service_context.service_name, resource_name, collection_name433            )434 435        collection_cls = type(str(cls_name), (ResourceCollection,), attrs)436 437        # Add the documentation to the collection manager's methods438        self._load_documented_collection_methods(439            attrs=attrs,440            resource_name=resource_name,441            collection_model=collection_model,442            service_model=service_context.service_model,443            event_emitter=event_emitter,444            base_class=CollectionManager,445        )446        attrs['_collection_cls'] = collection_cls447        cls_name += 'Manager'448 449        return type(str(cls_name), (CollectionManager,), attrs)450 451    def _load_batch_actions(452        self,453        attrs,454        resource_name,455        collection_model,456        service_model,457        event_emitter,458    ):459        """460        Batch actions on the collection become methods on both461        the collection manager and iterators.462        """463        for action_model in collection_model.batch_actions:464            snake_cased = xform_name(action_model.name)465            attrs[snake_cased] = self._create_batch_action(466                resource_name,467                snake_cased,468                action_model,469                collection_model,470                service_model,471                event_emitter,472            )473 474    def _load_documented_collection_methods(475        factory_self,476        attrs,477        resource_name,478        collection_model,479        service_model,480        event_emitter,481        base_class,482    ):483        # The base class already has these methods defined. However484        # the docstrings are generic and not based for a particular service485        # or resource. So we override these methods by proxying to the486        # base class's builtin method and adding a docstring487        # that pertains to the resource.488 489        # A collection's all() method.490        def all(self):491            return base_class.all(self)492 493        all.__doc__ = docstring.CollectionMethodDocstring(494            resource_name=resource_name,495            action_name='all',496            event_emitter=event_emitter,497            collection_model=collection_model,498            service_model=service_model,499            include_signature=False,500        )501        attrs['all'] = all502 503        # The collection's filter() method.504        def filter(self, **kwargs):505            return base_class.filter(self, **kwargs)506 507        filter.__doc__ = docstring.CollectionMethodDocstring(508            resource_name=resource_name,509            action_name='filter',510            event_emitter=event_emitter,511            collection_model=collection_model,512            service_model=service_model,513            include_signature=False,514        )515        attrs['filter'] = filter516 517        # The collection's limit method.518        def limit(self, count):519            return base_class.limit(self, count)520 521        limit.__doc__ = docstring.CollectionMethodDocstring(522            resource_name=resource_name,523            action_name='limit',524            event_emitter=event_emitter,525            collection_model=collection_model,526            service_model=service_model,527            include_signature=False,528        )529        attrs['limit'] = limit530 531        # The collection's page_size method.532        def page_size(self, count):533            return base_class.page_size(self, count)534 535        page_size.__doc__ = docstring.CollectionMethodDocstring(536            resource_name=resource_name,537            action_name='page_size',538            event_emitter=event_emitter,539            collection_model=collection_model,540            service_model=service_model,541            include_signature=False,542        )543        attrs['page_size'] = page_size544 545    def _create_batch_action(546        factory_self,547        resource_name,548        snake_cased,549        action_model,550        collection_model,551        service_model,552        event_emitter,553    ):554        """555        Creates a new method which makes a batch operation request556        to the underlying service API.557        """558        action = BatchAction(action_model)559 560        def batch_action(self, *args, **kwargs):561            return action(self, *args, **kwargs)562 563        batch_action.__name__ = str(snake_cased)564        batch_action.__doc__ = docstring.BatchActionDocstring(565            resource_name=resource_name,566            event_emitter=event_emitter,567            batch_action_model=action_model,568            service_model=service_model,569            collection_model=collection_model,570            include_signature=False,571        )572        return batch_action573