codekingpro/portable-devtools

1from __future__ import print_function2 3import collections4from contextlib import contextmanager5 6from eventlet import queue7 8 9__all__ = ['Pool', 'TokenPool']10 11 12class Pool(object):13    """14    Pool class implements resource limitation and construction.15 16    There are two ways of using Pool: passing a `create` argument or17    subclassing. In either case you must provide a way to create18    the resource.19 20    When using `create` argument, pass a function with no arguments::21 22        http_pool = pools.Pool(create=httplib2.Http)23 24    If you need to pass arguments, build a nullary function with either25    `lambda` expression::26 27        http_pool = pools.Pool(create=lambda: httplib2.Http(timeout=90))28 29    or :func:`functools.partial`::30 31        from functools import partial32        http_pool = pools.Pool(create=partial(httplib2.Http, timeout=90))33 34    When subclassing, define only the :meth:`create` method35    to implement the desired resource::36 37        class MyPool(pools.Pool):38            def create(self):39                return MyObject()40 41    If using 2.5 or greater, the :meth:`item` method acts as a context manager;42    that's the best way to use it::43 44        with mypool.item() as thing:45            thing.dostuff()46 47    The maximum size of the pool can be modified at runtime via48    the :meth:`resize` method.49 50    Specifying a non-zero *min-size* argument pre-populates the pool with51    *min_size* items.  *max-size* sets a hard limit to the size of the pool --52    it cannot contain any more items than *max_size*, and if there are already53    *max_size* items 'checked out' of the pool, the pool will cause any54    greenthread calling :meth:`get` to cooperatively yield until an item55    is :meth:`put` in.56    """57 58    def __init__(self, min_size=0, max_size=4, order_as_stack=False, create=None):59        """*order_as_stack* governs the ordering of the items in the free pool.60        If ``False`` (the default), the free items collection (of items that61        were created and were put back in the pool) acts as a round-robin,62        giving each item approximately equal utilization.  If ``True``, the63        free pool acts as a FILO stack, which preferentially re-uses items that64        have most recently been used.65        """66        self.min_size = min_size67        self.max_size = max_size68        self.order_as_stack = order_as_stack69        self.current_size = 070        self.channel = queue.LightQueue(0)71        self.free_items = collections.deque()72        if create is not None:73            self.create = create74 75        for x in range(min_size):76            self.current_size += 177            self.free_items.append(self.create())78 79    def get(self):80        """Return an item from the pool, when one is available.  This may81        cause the calling greenthread to block.82        """83        if self.free_items:84            return self.free_items.popleft()85        self.current_size += 186        if self.current_size <= self.max_size:87            try:88                created = self.create()89            except:90                self.current_size -= 191                raise92            return created93        self.current_size -= 1  # did not create94        return self.channel.get()95 96    @contextmanager97    def item(self):98        """ Get an object out of the pool, for use with with statement.99 100        >>> from eventlet import pools101        >>> pool = pools.TokenPool(max_size=4)102        >>> with pool.item() as obj:103        ...     print("got token")104        ...105        got token106        >>> pool.free()107        4108        """109        obj = self.get()110        try:111            yield obj112        finally:113            self.put(obj)114 115    def put(self, item):116        """Put an item back into the pool, when done.  This may117        cause the putting greenthread to block.118        """119        if self.current_size > self.max_size:120            self.current_size -= 1121            return122 123        if self.waiting():124            try:125                self.channel.put(item, block=False)126                return127            except queue.Full:128                pass129 130        if self.order_as_stack:131            self.free_items.appendleft(item)132        else:133            self.free_items.append(item)134 135    def resize(self, new_size):136        """Resize the pool to *new_size*.137 138        Adjusting this number does not affect existing items checked out of139        the pool, nor on any greenthreads who are waiting for an item to free140        up.  Some indeterminate number of :meth:`get`/:meth:`put`141        cycles will be necessary before the new maximum size truly matches142        the actual operation of the pool.143        """144        self.max_size = new_size145 146    def free(self):147        """Return the number of free items in the pool.  This corresponds148        to the number of :meth:`get` calls needed to empty the pool.149        """150        return len(self.free_items) + self.max_size - self.current_size151 152    def waiting(self):153        """Return the number of routines waiting for a pool item.154        """155        return max(0, self.channel.getting() - self.channel.putting())156 157    def create(self):158        """Generate a new pool item.  In order for the pool to159        function, either this method must be overriden in a subclass160        or the pool must be constructed with the `create` argument.161        It accepts no arguments and returns a single instance of162        whatever thing the pool is supposed to contain.163 164        In general, :meth:`create` is called whenever the pool exceeds its165        previous high-water mark of concurrently-checked-out-items.  In other166        words, in a new pool with *min_size* of 0, the very first call167        to :meth:`get` will result in a call to :meth:`create`.  If the first168        caller calls :meth:`put` before some other caller calls :meth:`get`,169        then the first item will be returned, and :meth:`create` will not be170        called a second time.171        """172        raise NotImplementedError("Implement in subclass")173 174 175class Token(object):176    pass177 178 179class TokenPool(Pool):180    """A pool which gives out tokens (opaque unique objects), which indicate181    that the coroutine which holds the token has a right to consume some182    limited resource.183    """184 185    def create(self):186        return Token()187