Python gevent.spawn() Examples

def test_whatever_1(self):
        """
        From a writing child, fire into the pipe. In a greenlet in the parent,
        receive one of these messages and return it to the main greenlet.
        Expect message retrieval (child process creation) within a certain
        timeout interval. Terminate the child process after retrieval.
        """
        with pipe() as (r, w):
            def readgreenlet(reader):
                with gevent.Timeout(SHORTTIME * 5, False) as t:
                    m = reader.get(timeout=t)
                    return m
            p = start_process(usecase_child_a, args=(w, ))
            # Wait for process to send first message:
            r.get()
            # Second message must be available immediately now.
            g = gevent.spawn(readgreenlet, r)
            m = r.get()
            assert g.get() == "SPLASH"
            p.terminate()
            p.join()
            assert p.exitcode == -signal.SIGTERM 

def resolve_hostnames(hostnames, timeout):
    """
    Do DNS resolution for a given list of hostnames

    This function uses gevent to resolve all the hostnames in *parallel*

    Args:
        hostnames (list): A list of strings
        timeout (int): The number of seconds to wait for resolution of **all** hostnames

    Returns:
        list: A list of (hostname, address) tuples in the same order as the input list of hostnames

    """
    assert validators.PanoptesValidators.valid_nonempty_iterable_of_strings(hostnames), u'hostnames should be a list'
    assert validators.PanoptesValidators.valid_nonzero_integer(timeout), u'timeout should be an int greater than zero'

    jobs = [gevent.spawn(wrap_errors(gaierror, socket.gethostbyname), host) for host in hostnames]
    gevent.joinall(jobs, timeout=timeout)
    addresses = [job.value if not isinstance(job.get(), gaierror) else None for job in jobs]
    results = [(hostnames[i], result) for i, result in enumerate(addresses)]
    return results 

def get_hostnames(ips, timeout):
    """
    Do DNS resolution for a given list of IPs

    Args:
        ips (list): A list of IPs
        timeout (int): The number of seconds to wait for resolution of **all** IPs

    Returns:
        list: A list of (address, hosname) tuples in the same order as the input list of IPs
    """
    assert validators.PanoptesValidators.valid_nonempty_iterable_of_strings(ips), u'ips should be a list'
    assert validators.PanoptesValidators.valid_nonzero_integer(timeout), u'timeout should be an int greater than zero'

    jobs = [gevent.spawn(wrap_errors((gaierror, herror), socket.gethostbyaddr), ip) for ip in ips]
    gevent.joinall(jobs, timeout=timeout)
    hostnames = [None if isinstance(job.get(), (gaierror, herror)) else job.value for job in jobs]
    results = {
        ips[i]: unknown_hostname(ips[i]) if ((not result) or
                                             (not result[0]) or
                                             result[0].startswith(u'UNKNOWN'))
        else result[0]
        for i, result in enumerate(hostnames)}
    return results 

def _wait_worker_ready(proc, resource_ref):
        worker_ips = []

        def waiter():
            check_time = time.time()
            while True:
                if not resource_ref.get_workers_meta():
                    gevent.sleep(0.1)
                    if proc.poll() is not None:
                        raise SystemError('Worker dead. exit code %s' % proc.poll())
                    if time.time() - check_time > 20:
                        raise TimeoutError('Check meta_timestamp timeout')
                    continue
                else:
                    break
            val = resource_ref.get_workers_meta()
            worker_ips.extend(val.keys())

        gl = gevent.spawn(waiter)
        gl.join()
        return worker_ips[0] 

def testConcurrentSend(self):
        with create_actor_pool(address=True, n_process=4, distributor=AdminDistributor(4),
                               backend='gevent') as pool:
            ref1 = pool.create_actor(DummyActor, 0)

            def ref_send(ref, rg):
                p = []
                for i in range(*rg):
                    p.append(gevent.spawn(ref.send, ('send', ref1, 'add_ret', i)))
                self.assertEqual([f.get() for f in p], list(range(*rg)))

            n_ref = 20

            refs = [pool.create_actor(DummyActor, 0) for _ in range(n_ref)]

            ps = []
            for i in range(n_ref):
                r = (i * 100, (i + 1) * 100)
                refx = refs[i]
                ps.append(gevent.spawn(ref_send, refx, r))

            [p.get() for p in ps] 

def child_client_runner(server_address):
    """I am executed in a child process.

    Run many HTTP clients, each in its own greenlet. Each HTTP client
        - establishes a TCP connection to the server running in the parent
        - sends an HTTP request through it
        - reads the HTTP response and validates the response body
    """

    def get():
        body = request.urlopen('http://%s:%s/' % server_address).read()
        assert body == DUMMY_PAYLOAD

    t0 = time.time()
    clients = [gevent.spawn(get) for _ in range(N_HTTP_CLIENTS)]

    # Wait until all `get()` greenlet instances have completed.
    gevent.joinall(clients)
    duration = time.time() - t0
    print('%s HTTP clients served within %.2f s.' % (N_HTTP_CLIENTS, duration)) 

def test_lock_out_of_context_single(self):
        r, w = pipe()
        g = gevent.spawn(lambda r: r.get(), r)
        gevent.sleep(SHORTTIME)
        with raises(GIPCLocked):
            with r:
                pass
                # The context manager can't close `r`, as it is locked in `g`.
        g.kill(block=False)
        # Ensure killing via 'context switch', i.e. yield control to other
        # coroutines (otherwise the subsequent close attempt will fail with
        # `GIPCLocked` error).
        gevent.sleep(-1)
        # Close writer first. otherwise, `os.close(r._fd)` would block on Win.
        w.close()
        r.close() 

def test_lock_out_of_context_single(self):
        h1, h2 = pipe(True)
        g = gevent.spawn(lambda h: h.get(), h1)
        gevent.sleep(SHORTTIME)
        with raises(GIPCLocked):
            with h1:
                pass
                # Can't close h1 reader on exit, as it is locked in `g`.
        g.kill(block=False)
        # Ensure killing via 'context switch', i.e. yield control to other
        # coroutines (otherwise the subsequent close attempt may fail with
        # `GIPCLocked` error).
        gevent.sleep(-1)
        h2.close()  # Closes read and write handles of h2.
        assert h1._writer._closed
        assert not h1._reader._closed
        h1.close()  # Closes read handle, ignore that writer is already closed.
        assert h1._reader._closed 

def spawn_greenlet(self, func, args, kwargs, greenlet_name):
    """Returns a gevent.Greenlet which has been initialized with the correct
    greenlet-local-storage state.

    Args:
      * greenlet_name (str|None) - If non-None, assign this to the greenlet's
        name.
    """
    self.close_non_parent_step()

    to_run = [pgs._get_setter_on_spawn() for pgs in PerGreentletStateRegistry]

    current_step = self._step_stack[-1]
    def _runner():
      for fn in to_run:
        fn()
      try:
        return func(*args, **kwargs)
      finally:
        self.close_non_parent_step()
    ret = gevent.spawn(_runner)
    if greenlet_name is not None:
      ret.name = greenlet_name
    current_step.greenlets.append(ret)
    return ret 

def exec_job(self, job_group, description, args, data=None, callback=None, timeout=None):
        jobid = str(uuid.uuid4())

        glet = gevent.spawn(
            self._job_monitor_glet,
            job_group, jobid, description, args, data
        )
        if callback is not None:
            glet.link(callback)

        timeout_glet = None
        if timeout is not None:
            timeout_glet = gevent.spawn(self._job_timeout_glet, job_group, jobid, timeout)

        self.running_jobs[job_group][jobid] = _Job(glet=glet, timeout_glet=timeout_glet)

        return jobid 

def run(target, timeout=1):
    '''Helper for running a thread

    Args:
        target (function): function to run on a thread
        timeout (int): how long to wait for target to return
    Returns:
        data: result of the function
    '''
    last = None
    done = False

    while not done:
        g = spawn(target)
        g.join(timeout)

        while not g.successful():
            yield StreamNone(last)

        last = g.value
        if last is None:
            done = True
        else:
            yield last 

def init():
    global API_CONFIG_PATH
    global API_CONFIG_LOCK

    config_path = os.environ.get('MM_CONFIG', None)
    if config_path is None:
        LOG.critical('MM_CONFIG environment variable not set')
        raise RuntimeError('MM_CONFIG environment variable not set')

    if not os.path.isdir(config_path):
        config_path = os.path.dirname(config_path)

    # init global vars
    API_CONFIG_PATH = os.path.join(config_path, 'api')
    API_CONFIG_LOCK = filelock.FileLock(
        os.environ.get('API_CONFIG_LOCK', '/var/run/minemeld/api-config.lock')
    )

    _load_config(config_path)
    _load_auth_dbs(config_path)
    if config_path is not None:
        gevent.spawn(_config_monitor, config_path) 

def start_dispatching(self):
        for rfcc in self.rpc_fanout_clients_channels:
            g = gevent.spawn(self._ioloop, rfcc)
            self.ioloops.append(g)
            g.link_exception(self._ioloop_failure)

        for rpcc in self.rpc_server_channels.values():
            g = gevent.spawn(self._ioloop, rpcc)
            self.ioloops.append(g)
            g.link_exception(self._ioloop_failure)

        for schannel in self.sub_channels:
            g = gevent.spawn(self._sub_ioloop, schannel)
            self.ioloops.append(g)
            g.link_exception(self._ioloop_failure)

        for mwschannel in self.mw_sub_channels:
            g = gevent.spawn(self._ioloop, mwschannel)
            self.ioloops.append(g)
            g.link_exception(self._ioloop_failure) 

def test_lock_out_of_context_pair_4(self):
        with raises(GIPCLocked):
            with pipe(True) as (h1, h2):
                # Write more to pipe than pipe buffer can hold
                # (makes `put` block when there is no reader).
                # Buffer is quite large on Windows.
                gw1 = gevent.spawn(lambda h: h.put(LONGERTHANBUFFER), h1)
                gw2 = gevent.spawn(lambda h: h.put(LONGERTHANBUFFER), h2)
                gevent.sleep(SHORTTIME)
                # Context fails closing h2 writer, succeeds upon closing h2
                # reader. Proceeds closing h1 writer, fails, closes h1
                # reader and succeeds.
        assert h2._reader._closed
        assert h1._reader._closed
        assert not h2._writer._closed
        assert not h1._writer._closed
        gw1.kill(block=False)
        gw2.kill(block=False)
        gevent.sleep(-1)
        h2.close()
        h1.close() 

def test_main(self):
        def main_task():
            with self.tracer.start_active_span('parent'):
                tasks = self.submit_callbacks()
                gevent.joinall(tasks)

        gevent.spawn(main_task)
        gevent.wait(timeout=5.0)

        spans = self.tracer.finished_spans()
        self.assertEquals(len(spans), 4)
        self.assertNamesEqual(spans, ['task', 'task', 'task', 'parent'])

        for i in range(3):
            self.assertSameTrace(spans[i], spans[-1])
            self.assertIsChildOf(spans[i], spans[-1]) 

def test_lock_out_of_context_pair_3(self):
        with raises(GIPCLocked):
            with pipe(True) as (h1, h2):
                gr1 = gevent.spawn(lambda h: h.get(), h1)
                gr2 = gevent.spawn(lambda h: h.get(), h2)
                gevent.sleep(SHORTTIME)
                # Context succeeds closing h2 writer, fails upon closing h2
                # reader. Proceeds closing h1 writer, succeeds, closes h1
                # reader and fails.
        assert not h2._reader._closed
        assert not h1._reader._closed
        assert h2._writer._closed
        assert h1._writer._closed
        gr1.kill(block=False)
        gr2.kill(block=False)
        gevent.sleep(-1)
        h2.close()
        h1.close() 

def submit(self):
        span = self.tracer.scope_manager.active.span

        def task1():
            with self.tracer.scope_manager.activate(span, False):
                span.set_tag('key1', '1')

                def task2():
                    with self.tracer.scope_manager.activate(span, False):
                        span.set_tag('key2', '2')

                        def task3():
                            with self.tracer.scope_manager.activate(span,
                                                                    True):
                                span.set_tag('key3', '3')

                        gevent.spawn(task3)

                gevent.spawn(task2)

        gevent.spawn(task1) 

def test_two_callbacks(self):
        response_greenlet1 = gevent.spawn(self.client.send_task, 'message1')
        response_greenlet2 = gevent.spawn(self.client.send_task, 'message2')

        gevent.joinall([response_greenlet1, response_greenlet2])

        self.assertEquals('message1::response', response_greenlet1.get())
        self.assertEquals('message2::response', response_greenlet2.get())

        spans = self.tracer.finished_spans()
        self.assertEquals(len(spans), 2)

        for span in spans:
            self.assertEquals(span.tags.get(tags.SPAN_KIND, None),
                              tags.SPAN_KIND_RPC_CLIENT)

        self.assertNotSameTrace(spans[0], spans[1])
        self.assertIsNone(spans[0].parent_id)
        self.assertIsNone(spans[1].parent_id) 

def usecase_child_d(forthreader, backwriter):
    recvqueue = gevent.queue.Queue()
    def g_from_forthpipe_to_q(forthreader):
        while True:
            m = forthreader.get()
            recvqueue.put(m)
            if m == "STOP":
                break

    def g_from_q_to_backpipe(backwriter):
        while True:
            m = recvqueue.get()
            backwriter.put(m)
            if m == "STOP":
                break

    g1 = gevent.spawn(g_from_forthpipe_to_q, forthreader)
    g2 = gevent.spawn(g_from_q_to_backpipe, backwriter)
    g1.get()
    g2.get() 

def __init__(self, name, truncate=False, bloom_filter_bits=0):
        if truncate:
            try:
                shutil.rmtree(name)
            except:
                pass

        self.db = None
        self._compact_glet = None

        self.db = plyvel.DB(
            name,
            create_if_missing=True,
            bloom_filter_bits=bloom_filter_bits
        )
        self._read_metadata()

        self.compact_interval = int(os.environ.get('MM_TABLE_COMPACT_INTERVAL', 3600 * 6))
        self.compact_delay = int(os.environ.get('MM_TABLE_COMPACT_DELAY', 3600))
        self._compact_glet = gevent.spawn(self._compact_loop) 

def enable_prevent_write(locker, timeout=900):
    global PREVENT_WRITE

    def _cleanup_prevent_write():
        gevent.sleep(timeout)
        LOG.info('Checking if prevent write still enabled by locker {}'.format(locker))
        disable_prevent_write(locker)

    with PREVENT_WRITE_GUARD:
        if PREVENT_WRITE is None:
            PREVENT_WRITE = locker
            gevent.spawn(_cleanup_prevent_write)

    return False 

def start(self):
        super(PanOSLogsAPIFT, self).start()

        if self.glet is not None:
            return

        self.glet = gevent.spawn_later(random.randint(0, 2), self._run)

        for idx in range(len(self.fields)):
            self.age_out_glets.append(
                gevent.spawn(self._age_out_loop, idx)
            ) 

def start_status_monitor(self):
        """Starts status monitor greenlet.
        """
        if self.status_glet is not None:
            LOG.error('double call to start_status')
            return

        self.status_glet = gevent.spawn(self._status_loop) 

def restart_minemeld_engine():
    info = MMSupervisor.supervisor.getProcessInfo('minemeld-engine')
    if info['statename'] == 'STARTING' or info['statename'] == 'STOPPING':
        return jsonify(error={
            'message': ('minemeld-engine not in RUNNING state: %s' %
                        info['statename'])
        }), 400

    gevent.spawn(_restart_engine)

    return jsonify(result='OK') 

def __init__(self, auto_refresh=False, load_all=False):
        self.load_all = load_all
        self.id_rule_map = {}
        strategys.load_strategys()
        self.load_rules()
        if auto_refresh:
            gevent.spawn(self.refresh) 

def send(r, pool=None, stream=False):
    if pool != None:
        return pool.spawn(r.send, stream=stream)
    return gevent.spawn(r.send, stream=stream) 

def test_multi_duplex(self):
        def duplex():
            with pipe() as (r, w):
                with pipe() as (r2, w2):
                    p = start_process(complchild_test_multi_duplex, (r, w2))
                    w.put("msg")
                    assert r2.get() == "msg"
                    p.join()
                    assert p.exitcode == 0

        duplexlets = [gevent.spawn(duplex) for _ in range(10)]
        for g in duplexlets:
            g.get() 

def handle_quit(self, sig, frame):
        # Move this out of the signal handler so we can use
        # blocking calls. See #1126
        gevent.spawn(super(GeventWorker, self).handle_quit, sig, frame) 

def patch_greenlet(func):
    """
    :param func:
    :return:
    """

    def wrapper(*args, **kwargs):
        return gevent.spawn(func, *args, **kwargs)

    return wrapper 

def loop():
    """
    :return:
    """
    gevent.joinall([gevent.spawn(master)])