Python gevent.wait() Examples

def test_main(self):
        def main_task():
            with self.tracer.start_active_span('parent'):
                tasks = self.submit_callbacks()
                gevent.joinall(tasks)

        gevent.spawn(main_task)
        gevent.wait(timeout=5.0)

        spans = self.tracer.finished_spans()
        self.assertEquals(len(spans), 4)
        self.assertNamesEqual(spans, ['task', 'task', 'task', 'parent'])

        for i in range(3):
            self.assertSameTrace(spans[i], spans[-1])
            self.assertIsChildOf(spans[i], spans[-1]) 

def finish(self, timeout=5):
        greenlets = []
        self._logs.irl.debug("Entering finish for amqp-stream monitor with %d trackers", len(self.__trackers))
        for tracker in self.__trackers.values():
            ttgls = tracker.start_finish(timeout=timeout)
            self._logs.irl.debug("  located %s greenlets (%s) in tracker %s", len(ttgls), tracker, ttgls)
            greenlets.extend(ttgls)
        self._logs.irl.debug("START wait for %d greenlets (%s)", len(greenlets), greenlets)
        gevent.wait(greenlets)
        reses = []
        self._logs.irl.debug("END wait for %d greenlets (%s)", len(greenlets), greenlets)
        for gr in greenlets:
            assert gr.ready(), \
                'all greenlets said they completed, but this one is not {}'.format(gr)
            if not gr.successful():
                raise gr.exception
            assert gr.successful(), \
                'a greenlet {} failed with {}.'.format(gr, gr.exception)
            results = gr.value
            reses.append(results)
            self._logs.irl.debug("  added results %s for greenlet %s", results, gr)
        self._logs.irl.debug("complete set of results for finish: %s", reses)
        return reses 

def test_main(self):
        # Create a Span and use it as (explicit) parent of a pair of subtasks.
        parent_span = self.tracer.start_span('parent')
        self.submit_subtasks(parent_span)

        gevent.wait(timeout=5.0)

        # Late-finish the parent Span now.
        parent_span.finish()

        spans = self.tracer.finished_spans()
        self.assertEqual(len(spans), 3)
        self.assertNamesEqual(spans, ['task1', 'task2', 'parent'])

        for i in range(2):
            self.assertSameTrace(spans[i], spans[-1])
            self.assertIsChildOf(spans[i], spans[-1])
            self.assertTrue(spans[i].finish_time <= spans[-1].finish_time)

    # Fire away a few subtasks, passing a parent Span whose lifetime
    # is not tied at all to the children. 

def test_main(self):
        # Start an isolated task and query for its result -and finish it-
        # in another task/thread
        span = self.tracer.start_span('initial')
        self.submit_another_task(span)

        gevent.wait(timeout=5.0)

        spans = self.tracer.finished_spans()
        self.assertEqual(len(spans), 3)
        self.assertNamesEqual(spans, ['initial', 'subtask', 'task'])

        # task/subtask are part of the same trace,
        # and subtask is a child of task
        self.assertSameTrace(spans[1], spans[2])
        self.assertIsChildOf(spans[1], spans[2])

        # initial task is not related in any way to those two tasks
        self.assertNotSameTrace(spans[0], spans[1])
        self.assertEqual(spans[0].parent_id, None) 

def wait(futures, timeout=None, count=None):
    """Blocks until `count` `futures` are done (or timeout occurs) then
    returns the list of done futures.

    This is analogous to `gevent.wait`.

    Args:
      * futures (List[Future]) - The Future objects to wait for.
      * timeout (None|seconds) - How long to wait for the Futures to be done.
        If we hit the timeout, wait will return even if we haven't reached
        `count` Futures yet.
      * count (None|int) - The number of Futures to wait to be done. If None,
        waits for all of them.

    Returns the list of done Futures, in the order in which they were done.
    """
    return list(_IWaitWrapper(futures, timeout, count)) 

def _reap_workers(workers, to_close, debug_log):
    """Collects the IO workers created with _mk_workers.

    After killing the workers, also closes the subprocess's open PIPE handles.

    See _safe_close for caveats around closing handles on windows.

    Args:
      * workers (List[Greenlet]) - The IO workers to kill.
      * to_close (List[...]) - (see _mk_workers for definition). The handles to
        close. These originate from the `Popen.std{out,err}` handles when the
        recipe engine had to use PIPEs.
      * debug_log (..stream.StreamEngine.Stream)

    Should not raise an exception.
    """
    debug_log.write_line('reaping IO workers...')
    for worker in workers:
      worker.kill()
    gevent.wait(workers)
    debug_log.write_line('  done')
    for handle_name, handle in to_close:
      _safe_close(debug_log, handle_name, handle) 

def sockets(redis):
    from dallinger.experiment_server import sockets

    sockets.redis_conn = redis
    # use a separate ChatBackend for each test
    sockets.chat_backend = sockets.ChatBackend()

    yield sockets

    # make sure all greenlets complete
    gevent.wait() 

def read_buffers_into_queue(
    stdout_buffer, stderr_buffer, timeout, print_output, print_prefix,
):
    output_queue = Queue()

    # Iterate through outputs to get an exit status and generate desired list
    # output, done in two greenlets so stdout isn't printed before stderr. Not
    # attached to state.pool to avoid blocking it with 2x n-hosts greenlets.
    stdout_reader = gevent.spawn(
        read_buffer,
        'stdout',
        stdout_buffer,
        output_queue,
        print_output=print_output,
        print_func=lambda line: '{0}{1}'.format(print_prefix, line),
    )
    stderr_reader = gevent.spawn(
        read_buffer,
        'stderr',
        stderr_buffer,
        output_queue,
        print_output=print_output,
        print_func=lambda line: '{0}{1}'.format(
            print_prefix, click.style(line, 'red'),
        ),
    )

    # Wait on output, with our timeout (or None)
    greenlets = gevent.wait((stdout_reader, stderr_reader), timeout=timeout)

    # Timeout doesn't raise an exception, but gevent.wait returns the greenlets
    # which did complete. So if both haven't completed, we kill them and fail
    # with a timeout.
    if len(greenlets) != 2:
        stdout_reader.kill()
        stderr_reader.kill()

        raise timeout_error()

    return list(output_queue.queue) 

def test_subscribes_to_redis(self, sockets, pubsub):
        sockets.Channel("custom").start()
        gevent.wait()
        pubsub.subscribe.assert_called_once_with([b"custom"]) 

def test_listen(self, sockets):
        sockets.redis_conn.pubsub.return_value = pubsub = Mock()
        pubsub.listen.return_value = [
            {"type": "message", "channel": b"quorum", "data": b"Calloo! Callay!"}
        ]

        channel = sockets.Channel("custom")
        client = Mock()
        channel.subscribe(client)
        channel.start()
        gevent.wait()  # wait for event loop

        client.send.assert_called_once_with("quorum:Calloo! Callay!") 

def test_heartbeat(self, sockets, client):
        client.ws.closed = False
        sockets.HEARTBEAT_DELAY = 1

        gevent.spawn(client.heartbeat)
        gevent.sleep(2)
        client.ws.closed = True
        gevent.wait()

        client.ws.send.assert_called_with("ping") 

def wait(self):
        gevent.wait() 

def main():
    args = sys.argv[1:]
    if len(args) != 2:
        sys.exit('Usage: %s source-address destination-address' % __file__)
    source = args[0]
    dest = parse_address(args[1])
    server = PortForwarder(source, dest)
    log('Starting port forwarder %s:%s -> %s:%s', *(server.address[:2] + dest))
    gevent.signal(signal.SIGTERM, server.close)
    gevent.signal(signal.SIGINT, server.close)
    server.start()
    gevent.wait() 

def wait(self, tasks, timeout=None, count=None):
        """
        Wait for tasks to finish with optional timeout.
        """

        assert isinstance(tasks, list)
        assert len(tasks) > 0

        gevent.wait(tasks, timeout, count) 

def __sync_issues(self, project):
        """
        同步 sonar 的 issue
        :return:
        """

        project.logger.info('[SonarScanner] Start sync sonar issues...')
        try:
            for issue_list in self.sonarqube.get_issues(project_key=project.key):
                for issue in issue_list:
                    self.__create_issue(project, issue)
                    # self.pool.spawn(self.__create_issue, project, issue)  #
                # gevent.wait()
        except Exception as ex:
            project.logger.error(ex) 

def handle_stop_test(self, test):
        """
        shutdown the little relay thing
        """
        self.__relay_queue.put('DONE-SPECIAL-MESSAGE')
        gevent.wait([self.__relay_greenlet]) 

def main():
    parser = _argparser()
    args = parser.parse_args()

    central = LE_Central(adapter=args.interface)

    connection = Connection(central)
    connection.start()

    code.InteractiveConsole.runsource = runsource_with_connection(connection)
    Thread(target=code.interact).start()

    gevent.wait() 

def run_local_process(
    command,
    stdin=None,
    timeout=None,
    print_output=False,
    print_prefix=None,
):
    process = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE)

    if stdin:
        write_stdin(stdin, process.stdin)

    combined_output = read_buffers_into_queue(
        process.stdout,
        process.stderr,
        timeout=timeout,
        print_output=print_output,
        print_prefix=print_prefix,
    )

    logger.debug('--> Waiting for exit status...')
    process.wait()
    logger.debug('--> Command exit status: {0}'.format(process.returncode))

    # Close any open file descriptors
    process.stdout.close()
    process.stderr.close()

    return process.returncode, combined_output 

def _kill(proc, gid):
    """Kills the process in group `gid` as gracefully as possible:

      * Send SIGTERM to the process group.
        * This is a bit atypical for POSIX, but this is done to provide
          consistent behavior between *nix/Windows (where we MUST signal the
          whole group). This allows writing parent/child programs which run
          cross-platform without requiring per-platform parent/child handling
          code.
        * If programs really don't want this, they should make their own process
          group for their children.
      * Give main process 30s to quit.
      * Send SIGKILL to the whole group (to avoid leaked processes). This will
        kill the process we spawned directly.

    Returns the process's returncode.
    """
    try:
      os.killpg(gid, signal.SIGTERM)
    except OSError:
      pass
    # TODO(iannucci): This API changes in python3 to raise an exception on
    # timeout.
    proc.wait(timeout=30)
    try:
      os.killpg(gid, signal.SIGKILL)
    except OSError:
      pass
    return proc.wait() 

def _kill(proc, gid):
    """Kills the process as gracefully as possible:

      * Send CTRL_BREAK_EVENT (to the process group, there's no other way)
      * Give main process 30s to quit.
      * Call TerminateProcess on the process we spawned.

    Unfortunately, we don't have a mechanism to do 'TerminateProcess' to the
    process's group, so this could leak processes on Windows.

    Returns the process's returncode.
    """
    # TODO(iannucci): Use a Job Object for process management. Use subprocess42
    # for reference.
    del gid  # gid is not supported on Windows
    try:
      # pylint: disable=no-member
      proc.send_signal(signal.CTRL_BREAK_EVENT)
    except OSError:
      pass
    # TODO(iannucci): This API changes in python3 to raise an exception on
    # timeout.
    proc.wait(timeout=30)
    if proc.returncode is not None:
      return proc.returncode
    try:
      proc.terminate()
    except OSError:
      pass
    return proc.wait() 

def exception(self, timeout=None):
      """Blocks until this Future is done, then returns (not raises) this
      Future's exception (or None if there was no exception).

      Args:
        * timeout (None|seconds) - How long to wait for the Future to be done.

      Returns the exception instance which would be raised from `result` if
      the Future is Done, otherwise None.

      Raises Timeout if the Future is not done within the given timeout.
      """
      with gevent.Timeout(timeout, exception=FuturesApi.Timeout()):
        done = gevent.wait([self._greenlet])[0]
        return done.exception 

def result(self, timeout=None):
      """Blocks until this Future is done, then returns its value, or raises
      its exception.

      Args:
        * timeout (None|seconds) - How long to wait for the Future to be done.

      Returns the result if the Future is done.

      Raises the Future's exception, if the Future is done with an error.

      Raises Timeout if the Future is not done within the given timeout.
      """
      with gevent.Timeout(timeout, exception=FuturesApi.Timeout()):
        return self._greenlet.get() 

def test(self):
        client = Client(self.tracer, self.queue)
        gevent.spawn(self.server.run)
        gevent.spawn(client.send)

        gevent.wait(timeout=5.0)

        spans = self.tracer.finished_spans()
        self.assertIsNotNone(get_one_by_tag(spans,
                                            tags.SPAN_KIND,
                                            tags.SPAN_KIND_RPC_SERVER))
        self.assertIsNotNone(get_one_by_tag(spans,
                                            tags.SPAN_KIND,
                                            tags.SPAN_KIND_RPC_CLIENT)) 

def test_main(self):
        # Start a Span and let the callback-chain
        # finish it when the task is done
        with self.tracer.start_active_span('one', finish_on_close=False):
            self.submit()

        gevent.wait()

        spans = self.tracer.finished_spans()
        self.assertEqual(len(spans), 1)
        self.assertEqual(spans[0].operation_name, 'one')

        for i in range(1, 4):
            self.assertEqual(spans[0].tags.get('key%s' % i, None), str(i)) 

def _work(self, burst=False, logging_level=logging.INFO):
        """Starts the work loop.

        Pops and performs all jobs on the current list of queues.  When all
        queues are empty, block and wait for new jobs to arrive on any of the
        queues, unless `burst` mode is enabled.

        The return value indicates whether any jobs were processed.
        """
        setup_loghandlers(logging_level)
        self._install_signal_handlers()

        self.did_perform_work = False
        self.register_birth()
        self.log.info(
            "RQ GEVENT worker (Greenlet pool size={0}) {1!r} started, version {2}".format(
                self.gevent_pool.size, self.key, VERSION
            )
        )
        self.set_state(WorkerStatus.STARTED)

        try:
            while True:
                try:
                    self.check_for_suspension(burst)

                    if self.should_run_maintenance_tasks:
                        self.clean_registries()

                    if self._stop_requested:
                        self.log.info("Stopping on request.")
                        break

                    timeout = None if burst else max(1, self.default_worker_ttl - 60)

                    result = self.dequeue_job_and_maintain_ttl(timeout)
                    if result is None and burst:
                        self.log.info("RQ worker {0!r} done, quitting".format(self.key))

                        try:
                            # Make sure dependented jobs are enqueued.
                            gevent.wait(self.children)
                        except LoopExit:
                            pass
                        result = self.dequeue_job_and_maintain_ttl(timeout)

                    if result is None:
                        break
                except StopRequested:
                    break

                job, queue = result
                self.execute_job(job, queue)

        finally:
            if not self.is_horse:
                self.register_death()
        return self.did_perform_work 

def _wait_proc(proc, gid, timeout, debug_log):
    """Waits for the completion (or timeout) of `proc`.

    Args:

      * proc (subprocess.Popen) - The actual running subprocess to wait for.
      * gid (int|None) - The group ID of the process.
      * timeout (Number|None) - The number of seconds to wait for the process to
        end (or None for no timeout).
      * debug_log (..stream.StreamEngine.Stream)

    Returns the ExecutionResult.

    Should not raise an exception.
    """
    ret = ExecutionResult()

    # We're about to do gevent-blocking operations (waiting on the subprocess)
    # and so another greenlet could kill us; we guard all of these operations
    # with a `try/except GreenletExit` to handle this and return an
    # ExecutionResult(was_cancelled=True) in that case.
    #
    # The engine will raise a new GreenletExit exception after processing this
    # step.
    try:
      # TODO(iannucci): This API changes in python3 to raise an exception on
      # timeout.
      proc.wait(timeout)
      ret = attr.evolve(ret, retcode=proc.poll())
      debug_log.write_line(
          'finished waiting for process, retcode %r' % ret.retcode)

      # TODO(iannucci): Make leaking subprocesses explicit (e.g. goma compiler
      # daemon). Better, change deamons to be owned by a gevent Greenlet (so that
      # we don't need to leak processes ever).
      #
      # _kill(proc, gid)  # In case of leaked subprocesses or timeout.
      if ret.retcode is None:
        debug_log.write_line('timeout! killing process group %r' % gid)
        # Process timed out, kill it. Currently all uses of non-None timeout
        # intend to actually kill the subprocess when the timeout pops.
        ret = attr.evolve(ret, retcode=_kill(proc, gid), had_timeout=True)

    except gevent.GreenletExit:
      debug_log.write_line(
          'caught GreenletExit, killing process group %r' % (gid,))
      ret = attr.evolve(ret, retcode=_kill(proc, gid), was_cancelled=True)

    return ret 

def iwait(futures, timeout=None, count=None):
    """Iteratively yield up to `count` Futures as they become done.


    This is analogous to `gevent.iwait`.

    Usage:

        for future in api.futures.iwait(futures):
          # consume future

    If you are not planning to consume the entire iwait iterator, you can
    avoid the resource leak by doing, for example:

        with api.futures.iwait(a, b, c) as iter:
          for future in iter:
            if future is a:
              break

    You might want to use `iwait` over `wait` if you want to process a group
    of Futures in the order in which they complete. Compare:

      for task in iwait(swarming_tasks):
        # task is done, do something with it

      vs

      while swarming_tasks:
        task = wait(swarming_tasks, count=1)[0]  # some task is done
        swarming_tasks.remove(task)
        # do something with it

    Args:
      * futures (List[Future]) - The Future objects to wait for.
      * timeout (None|seconds) - How long to wait for the Futures to be done.
      * count (None|int) - The number of Futures to yield. If None,
        yields all of them.

    Yields futures in the order in which they complete until we hit the
    timeout or count. May also be used with a context manager to avoid
    leaking resources if you don't plan on consuming the entire iterable.
    """
    return _IWaitWrapper(futures, timeout, count) 

def start(self):
        self.register_logger("gevent_helpers")
        for comp in self.components.itervalues():
            comp.manager = self  # Add a backreference for leaky abstractions
            comp.counters = self.register_stat_counters(comp, comp.one_min_stats, comp.one_sec_stats)
            # Register a separate logger for each component
            if comp is not self:
                comp.logger = self.register_logger(comp.name)
                comp.start()

        # Starts the greenlet
        Component.start(self)
        # Start the datagram control server if it's been inited
        if self.config['datagram']['enabled']:
            DatagramServer.start(self, )

        # This is the main thread of execution, so just continue here waiting
        # for exit signals
        ######
        # Register shutdown signals
        gevent.signal(signal.SIGUSR1, self.dump_objgraph)
        gevent.signal(signal.SIGHUP, exit, "SIGHUP")
        gevent.signal(signal.SIGINT, exit, "SIGINT")
        gevent.signal(signal.SIGTERM, exit, "SIGTERM")

        try:
            gevent.wait()
        # Allow a force exit from multiple exit signals (Ctrl+C mashed multiple times)
        finally:
            self.logger.info("Exiting requested, allowing {} seconds for cleanup."
                             .format(self.config['term_timeout']))
            try:
                for comp in self.components.itervalues():
                    self.logger.debug("Calling stop on component {}".format(comp))
                    comp.stop()
                if gevent.wait(timeout=self.config['term_timeout']):
                    self.logger.info("All threads exited normally")
                else:
                    self.logger.info("Timeout reached, shutting down forcefully")
            except gevent.GreenletExit:
                self.logger.info("Shutdown requested again by system, "
                                 "exiting without cleanup")
            self.logger.info("Exit")
            self.logger.info("=" * 80) 

def spawn(self, func, *args, **kwargs):
    """Prepares a Future to run `func(*args, **kwargs)` concurrently.

    Any steps executed in `func` will only have manipulable StepPresentation
    within the scope of the executed function.

    Because this will spawn a greenlet on the same OS thread (and not,
    for example a different OS thread or process), `func` can easily be an
    inner function, closure, lambda, etc. In particular, func, args and kwargs
    do not need to be pickle-able.

    This function does NOT switch to the greenlet (you'll have to block on a
    future/step for that to happen). In particular, this means that the
    following pattern is safe:

        # self._my_future check + spawn + assignment is atomic because
        # no switch points occur.
        if not self._my_future:
          self._my_future = api.futures.spawn(func)

    NOTE: If used in @@@annotator@@@ mode, this will block on the completion of
    the Future before returning it.

    Kwargs:

      * __name (str) - If provided, will assign this name to the spawned
        greenlet. Useful if this greenlet ends up raising an exception, this
        name will appear in the stderr logging for the engine. See
        `Future.name` for more information.
      * __meta (any) - If provided, will assign this metadata to the returned
        Future. This field is for your exclusive use.
      * Everything else is passed to `func`.

    Returns a Future of `func`'s result.
    """
    name = kwargs.pop('__name', None)
    if name is None:
      name = 'Future-%d' % (self._future_id,)
    self._future_id += 1

    meta = kwargs.pop('__meta', None)

    ret = self.Future(self.concurrency_client.spawn(
        func, args, kwargs, name), name, meta)
    if not self.concurrency_client.supports_concurrency: # pragma: no cover
      # test mode always supports concurrency, hence the nocover
      self.wait([ret])
    return ret 

def start_blacklist(self, project):
        """

        :param project:
        :return:
        """
        def __scan(file_path):
            for rule in self.blacklist_rule:
                flag, _ = rule.verify(reference_value=file_path)
                project.logger.debug("[RuleScanner] rule: [{0}], file: [{1}]".format(rule, file_path))
                if flag:
                    relative_path = file_path.replace(project.scan_path, "")
                    project.logger.info("[RuleScanner] [Blacklist] [+] Found '{0}' vulnerability in '{1}' file.".format(rule.name, relative_path))
                    info = rule.get_info(match_content=_, origin_file=file_path)
                    if project.web_url:
                        report = '{0}/blob/{1}{2}#L{3}'.format(
                            project.web_url,
                            project.branch,
                            relative_path,
                            info['start_line']
                        )
                    else:
                        report = ''
                    author, author_email = get_git_author(project.get_last_author(file_path))
                    vuln = Vulnerability(
                        task_id=project.task_id,
                        rule_key=rule.key,
                        risk_id=rule.risk_id,
                        title=rule.name,
                        file=relative_path,
                        author=author,
                        author_email=author_email,
                        hash=project.get_last_commit(),
                        start_line=info['start_line'],
                        end_line=info['end_line'],
                        report=report,
                        code_example=info['code_example'],
                        evidence_content=_,
                        engine=self.key,
                    )
                    vuln_key = hash_md5('{0}_{1}_{2}'.format(file_path, rule.id, info['start_line']))
                    if vuln_key not in kb.result[project.key]:
                        kb.result[project.key][vuln_key] = vuln.info
                        project.logger.debug('[RuleScanner] {0}'.format(vuln))

        project.logger.info("[RuleScanner] Begin to perform rule-based blacklist vulnerability analysis...")
        pool = ThreadPool(project.threads or 20)
        for fpath, dirs, fs in os.walk(project.scan_path):
            for f in fs:
                pool.spawn(__scan,  os.path.join(fpath, f))
            gevent.wait()

        project.logger.info("[RuleScanner] Rule blacklist scan completed.")