Python gevent.pool.spawn() Examples

The following are 7 code examples of gevent.pool.spawn(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module gevent.pool , or try the search function .
Example #1
Source File: data_main.py    From backtrader-cn with GNU General Public License v3.0 6 votes vote down vote up 
def download_delta_data(stocks, pool_size=40):
    """
    Download delta data for all stocks collections of all libraries.
    :param stocks: stock code list.
    :param pool_size: the pool size of gevent.pool.Pool.
    :return: None
    """

    pool = gevent.pool.Pool(pool_size)
    for i in range(len(stocks) // pool_size + 1):
        start = i * pool_size
        end = (i + 1) * pool_size
        lst = stocks[start:end]
        logger.debug(f'download delta data for stock list: {lst}')
        for stock in lst:
            pool.spawn(bdt.TsHisData.download_one_delta_data, stock)
        pool.join(timeout=30)
Example #2
Source File: async.py    From cascade-server with Apache License 2.0 6 votes vote down vote up 
def async_routine(f):
    @wraps(f)
    def to_list(*args, **kwargs):
        output = f(*args, **kwargs)
        if output is not None:
            return list(output)

    def async_wrapped_routine(*args, **kwargs):
        run_async = kwargs.pop('async', True)
        if enabled and run_async:
            logger.debug('Spawning greenlet to run {}({}) asynchronously'.format(f.func_name, args))
            return spawn(to_list, *args, **kwargs)
        else:
            logger.debug('Asynchronous support disabled. Running {}({}) in a thread'.format(f.func_name, args))
            to_list(*args, **kwargs)
    return async_wrapped_routine
Example #3
Source File: main.py    From satori with Apache License 2.0 5 votes vote down vote up 
def send_alarm(ev):
    backends = get_relevant_backends(ev)

    for p in backends:
        try:
            ev1 = copy.deepcopy(ev)
            rst = call_hooks('send_alarm', (ev1, p))
            if not rst:
                continue

            ev1, _ = rst
            log.debug('Sending alarm via backend %s', p.__class__.__name__)
            gevent.spawn(p.send, ev1)
        except Exception:
            log.exception('Error sending alarm')
Example #4
Source File: main.py    From satori with Apache License 2.0 5 votes vote down vote up 
def process_events():
    r = redis.from_url(State.config['redis'])
    queues = [
        'satori-events:%s' % i for i in range(15)
    ]

    while True:
        o, raw = r.blpop(queues)
        pool.spawn(process_single_event, json.loads(raw))
Example #5
Source File: session.py    From eppy with MIT License 5 votes vote down vote up 
def start(self):
        pool = gevent.pool.Pool(size=self.concurrency)
        try:
            for i in xrange(1, self.num_connectors + 1):
                pool.spawn(self.connector)
                time.sleep(self.spawn_interval)

            pool.join()
        except KeyboardInterrupt:
            pass
Example #6
Source File: scheduler.py    From cloud-volume with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
def schedule_green_jobs(
    fns, concurrency=DEFAULT_THREADS, 
    progress=None, total=None
  ):
  import gevent.pool

  if total is None:
    try:
      total = len(fns)
    except TypeError: # generators don't have len
      pass

  pbar = tqdm(total=total, desc=progress, disable=(not progress))
  results = []
  
  def updatefn(fn):
    def realupdatefn():
      res = fn()
      pbar.update(1)
      results.append(res)
    return realupdatefn

  pool = gevent.pool.Pool(concurrency)
  for fn in fns:
    pool.spawn( updatefn(fn) )

  pool.join()
  pool.kill()
  pbar.close()

  return results
Example #7
Source File: asyncnet.py    From pytest-concurrent with MIT License 5 votes vote down vote up 
def run_items(self, items, session, workers=None):
        import gevent
        import gevent.monkey
        import gevent.pool
        gevent.monkey.patch_all()
        pool = gevent.pool.Pool(size=workers)
        for index, item in enumerate(items):
            pool.spawn(self._run_next_item, session, item, index)
        pool.join()