Python gevent.pool.Group() Examples

def forward(self,sock,remote_sock,data_timeout=5*60):
        u"""在两个套接字之间转发数据(阻塞调用)

在转发失败时自动关闭连接。在双向都出现超时的情况下会关闭连接。

未专门处理 shutdown ，单方向 shutdown 时会关闭双向链接。

"""
        try:
            o = {
                # 最后一次转发数据的时间 = int(time()*1000)
                'forward_data_time':int(time.time()*1000),
            }
            sock.settimeout(data_timeout)
            remote_sock.settimeout(data_timeout)

            group = Group()
            group.add(gevent.spawn(self.__forwardData,sock,remote_sock,o,data_timeout))
            group.add(gevent.spawn(self.__forwardData,remote_sock,sock,o,data_timeout))
            group.join()
        finally:
            sock.close()
            remote_sock.close() 

def __init__(self, nsqd_tcp_addresses=[], max_backoff_duration=128,
                 **kwargs):
        if not nsqd_tcp_addresses:
            raise ValueError('must specify at least one nsqd or lookupd')

        self.nsqd_tcp_addresses = parse_nsqds(nsqd_tcp_addresses)
        self.max_backoff_duration = max_backoff_duration
        self.conn_kwargs = kwargs
        self.logger = logging.getLogger(__name__)

        self._state = INIT
        self._connections = Queue()
        self._connection_backoffs = defaultdict(self._create_backoff)
        self._response_queues = {}
        self._workers = Group() 

def spawn_later(self, seconds, func, *args, **kwargs):
        parent = super(GreenletGroupWithExceptionCatching, self)
        func_wrap = self._wrap_errors(func)
        # spawn_later doesn't exist in pool.Group, so let's implement it below
        greenlet = parent.greenlet_class(func_wrap, *args, **kwargs)
        parent.add(greenlet)
        greenlet.start_later(seconds)
        return greenlet 

def spawn_imap_unordered(self):
        igroup = Group()
        result = []
        with tracer.start_active_span('test'):
            for i in igroup.imap_unordered(self.make_http_call, range(3)):
                result.append(i) 

def parallel_map(func, iterable, args=None, kwargs=None, workers=None):
    """Map func on a list using gevent greenlets.

    :param func: function applied on iterable elements
    :type func: function
    :param iterable: elements to map the function over
    :type iterable: iterable
    :param args: arguments of func
    :type args: tuple
    :param kwargs: keyword arguments of func
    :type kwargs: dict
    :param workers: limit the number of greenlets
                    running in parrallel
    :type workers: int
    """
    if args is None:
        args = ()
    if kwargs is None:
        kwargs = {}
    if workers is not None:
        pool = Pool(workers)
    else:
        pool = Group()
    iterable = [pool.spawn(func, i, *args, **kwargs) for i in iterable]
    pool.join(raise_error=True)
    for idx, i in enumerate(iterable):
        i_type = type(i.get())
        i_value = i.get()
        if issubclass(i_type, BaseException):
            raise i_value
        iterable[idx] = i_value
    return iterable 

def _filter_transactions(self, transactions: List[Transaction]) -> List[Transaction]:
        """Filters all transactions in parallel. The results may not have the same order."""
        filter_task_group = pool.Group()
        filter_task_results = filter_task_group.imap_unordered(self._filter_transaction_task, transactions)
        return filter_array(lambda el: el is not None, filter_task_results) 

def __init__(self, account,
                 heartbeat=1, refresh_frequency=30):
        self.refresh_frequency = refresh_frequency
        self.syncmanager_lock = BoundedSemaphore(1)
        self.saved_remote_folders = None
        self.sync_engine_class = FolderSyncEngine

        self.folder_monitors = Group()
        self.delete_handler = None

        BaseMailSyncMonitor.__init__(self, account, heartbeat) 

def __init__(self, topic, channel, nsqd_tcp_addresses=[],
                 lookupd_http_addresses=[], name=None, message_handler=None,
                 max_tries=5, max_in_flight=1, requeue_delay=0,
                 lookupd_poll_interval=60, lookupd_poll_jitter=0.3,
                 low_ready_idle_timeout=10, max_backoff_duration=128,
                 backoff_on_requeue=True, **kwargs):
        if not nsqd_tcp_addresses and not lookupd_http_addresses:
            raise ValueError('must specify at least one nsqd or lookupd')

        self.nsqd_tcp_addresses = parse_nsqds(nsqd_tcp_addresses)
        self.lookupds = parse_lookupds(lookupd_http_addresses)
        self.iterlookupds = cycle(self.lookupds)

        self.topic = topic
        self.channel = channel
        self.max_tries = max_tries
        self.max_in_flight = max_in_flight
        self.requeue_delay = requeue_delay
        self.lookupd_poll_interval = lookupd_poll_interval
        self.lookupd_poll_jitter = lookupd_poll_jitter
        self.low_ready_idle_timeout = low_ready_idle_timeout
        self.backoff_on_requeue = backoff_on_requeue
        self.max_backoff_duration = max_backoff_duration
        self.conn_kwargs = kwargs

        if name:
            self.name = name
        else:
            self.name = '%s.%s.%s' % (__name__, self.topic, self.channel)

        if message_handler is not None:
            self.on_message.connect(message_handler, weak=False)

        self.logger = logging.getLogger(self.name)

        self._state = INIT
        self._redistributed_ready_event = Event()
        self._connection_backoffs = defaultdict(self._create_backoff)
        self._message_backoffs = defaultdict(self._create_backoff)

        self._connections = {}
        self._workers = Group()
        self._killables = Group() 

def create_connection(self, address, timeout=10):

        ip_list = dnslib.dnsQuery(address[0])

        # 尝试连接缓存
        route_list = self.get_route_order_ping(address[0],address[1],None)
        if route_list:
            try:
                route = route_list[0]
                hit_ip = route['hit_ip']

                if hit_ip in ip_list:
                    cache_timeout = route['tcp_ping']

                    if cache_timeout<1000:
                        cache_timeout = cache_timeout * 2
                    else:
                        cache_timeout = cache_timeout+1000
                    cache_timeout = int(math.ceil(cache_timeout/1000.0))

                    start_time = int(time.time() * 1000)
                    sock = self._direct_create_connection(address,hit_ip, cache_timeout)
                    t = int(time.time() * 1000) - start_time
                    logging.debug(u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s 命中。time:%s'%(self.get_display_name(),address[0],hit_ip,address[1],t))
                    self.update_route_ping(address[0],address[1],t,hit_ip)
                    return sock
                else:
                    logging.debug(u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s IP 不匹配，放弃缓存。'%(self.get_display_name(),address[0],hit_ip,address[1]))
            except:
                t = int(time.time() * 1000) - start_time
                info = traceback.format_exc()
                logging.debug(
                    u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s 失败。time:%s' % (self.get_display_name(), address[0],hit_ip, address[1],t))
                logging.debug('%s\r\n\r\n' % info)

        # 缓存失败，连接全部
        evt = Event()
        group = Group()
        aync_task = DirectAsyncTask(evt, None, group)

        for ip in ip_list:
            group.add(gevent.spawn(self._create_connection,  aync_task, address,ip, timeout))

        # 所有连接失败时发出通知
        gevent.spawn(self._create_connection_all_end, aync_task)

        evt.wait()
        if aync_task.sock:
            return aync_task.sock
        else:
            raise UpstreamConnectError() 

def create_connection(self, address, timeout=10):

        # 尝试连接缓存
        route_list = self.get_route_order_ping(address[0],address[1],None)
        if route_list:
            try:
                route = route_list[0]

                cache_timeout = route['tcp_ping']

                if cache_timeout<1000:
                    cache_timeout = cache_timeout * 2
                else:
                    cache_timeout = cache_timeout+1000
                cache_timeout = int(math.ceil(cache_timeout/1000.0))

                _upstream = self.upstream_dict.get(route['proxy_name'])
                start_time = int(time.time() * 1000)
                sock = _upstream.create_connection(address, cache_timeout)
                t = int(time.time() * 1000) - start_time
                logging.debug(u'[upstream][RouteCache]%s 缓存记录 连接 %s:%s 命中。time:%s'%(_upstream.get_display_name(),address[0],address[1],t))
                self.update_route_ping(_upstream.get_name(),address[0],address[1],t)
                return sock
            except:
                t = int(time.time() * 1000) - start_time
                info = traceback.format_exc()
                logging.debug(
                    u'[upstream][RouteCache]%s 缓存记录 连接 %s:%s 失败。time:%s' % (_upstream.get_display_name(), address[0], address[1],t))
                logging.debug('%s\r\n\r\n' % info)

        # 缓存失败，连接全部
        evt = Event()
        group = Group()
        aync_task = MultipathAsyncTask(evt, None, group)

        for _upstream in self.upstream_dict.values():
            group.add(gevent.spawn(self._create_connection, _upstream, aync_task, address, timeout))

        # 所有连接失败时发出通知
        gevent.spawn(self._create_connection_all_end, aync_task)

        evt.wait()
        if aync_task.sock:
            return aync_task.sock
        else:
            raise UpstreamConnectError()