Python gevent.joinall() Examples

def test_logged_no_rate_limit_because_remain_count(
        client, client_ip, test_user, test_token, mocker):
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
        test_user.username: {
            'rate': 100,
            'capacity': 300,
        }
    })

    def worker(queue):
        response = client.get(
            '/api/need_login', headers={
                'Authorization': test_token,
            })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def child_client_runner(server_address):
    """I am executed in a child process.

    Run many HTTP clients, each in its own greenlet. Each HTTP client
        - establishes a TCP connection to the server running in the parent
        - sends an HTTP request through it
        - reads the HTTP response and validates the response body
    """

    def get():
        body = request.urlopen('http://%s:%s/' % server_address).read()
        assert body == DUMMY_PAYLOAD

    t0 = time.time()
    clients = [gevent.spawn(get) for _ in range(N_HTTP_CLIENTS)]

    # Wait until all `get()` greenlet instances have completed.
    gevent.joinall(clients)
    duration = time.time() - t0
    print('%s HTTP clients served within %.2f s.' % (N_HTTP_CLIENTS, duration)) 

def test_main(self):
        def main_task():
            with self.tracer.start_active_span('parent'):
                tasks = self.submit_callbacks()
                gevent.joinall(tasks)

        gevent.spawn(main_task)
        gevent.wait(timeout=5.0)

        spans = self.tracer.finished_spans()
        self.assertEquals(len(spans), 4)
        self.assertNamesEqual(spans, ['task', 'task', 'task', 'parent'])

        for i in range(3):
            self.assertSameTrace(spans[i], spans[-1])
            self.assertIsChildOf(spans[i], spans[-1]) 

def get_hostnames(ips, timeout):
    """
    Do DNS resolution for a given list of IPs

    Args:
        ips (list): A list of IPs
        timeout (int): The number of seconds to wait for resolution of **all** IPs

    Returns:
        list: A list of (address, hosname) tuples in the same order as the input list of IPs
    """
    assert validators.PanoptesValidators.valid_nonempty_iterable_of_strings(ips), u'ips should be a list'
    assert validators.PanoptesValidators.valid_nonzero_integer(timeout), u'timeout should be an int greater than zero'

    jobs = [gevent.spawn(wrap_errors((gaierror, herror), socket.gethostbyaddr), ip) for ip in ips]
    gevent.joinall(jobs, timeout=timeout)
    hostnames = [None if isinstance(job.get(), (gaierror, herror)) else job.value for job in jobs]
    results = {
        ips[i]: unknown_hostname(ips[i]) if ((not result) or
                                             (not result[0]) or
                                             result[0].startswith(u'UNKNOWN'))
        else result[0]
        for i, result in enumerate(hostnames)}
    return results 

def resolve_hostnames(hostnames, timeout):
    """
    Do DNS resolution for a given list of hostnames

    This function uses gevent to resolve all the hostnames in *parallel*

    Args:
        hostnames (list): A list of strings
        timeout (int): The number of seconds to wait for resolution of **all** hostnames

    Returns:
        list: A list of (hostname, address) tuples in the same order as the input list of hostnames

    """
    assert validators.PanoptesValidators.valid_nonempty_iterable_of_strings(hostnames), u'hostnames should be a list'
    assert validators.PanoptesValidators.valid_nonzero_integer(timeout), u'timeout should be an int greater than zero'

    jobs = [gevent.spawn(wrap_errors(gaierror, socket.gethostbyname), host) for host in hostnames]
    gevent.joinall(jobs, timeout=timeout)
    addresses = [job.value if not isinstance(job.get(), gaierror) else None for job in jobs]
    results = [(hostnames[i], result) for i, result in enumerate(addresses)]
    return results 

def test_async_publish():
    with NsqdIntegrationServer() as server:
        results = []
        producer = Producer(server.tcp_address)
        producer.start()

        for _ in range(100):
            results.append(producer.publish('test', b'hi', raise_error=False))

        gevent.joinall(results, raise_error=True)
        producer.close()
        producer.join()

        conn = NsqdHTTPClient(server.address, server.http_port)
        stats = conn.stats()

        assert stats['topics'][0]['depth'] == 100 

def test_async_multipublish():
    with NsqdIntegrationServer() as server:
        results = []
        producer = Producer(server.tcp_address)
        producer.start()

        for _ in range(10):
            result = producer.multipublish(
                'test', 10 * [b'hi'], raise_error=False)
            results.append(result)

        gevent.joinall(results, raise_error=True)
        producer.close()
        producer.join()

        conn = NsqdHTTPClient(server.address, server.http_port)
        stats = conn.stats()

        assert stats['topics'][0]['depth'] == 100 

def _run_no_wait_ops(state):
    '''
    Run all ops for all servers at once.
    '''

    hosts_operations = product(state.inventory, state.get_op_order())
    with progress_spinner(hosts_operations) as progress:
        # Spawn greenlet for each host to run *all* ops
        greenlets = [
            state.pool.spawn(
                _run_server_ops, state, host,
                progress=progress,
            )
            for host in state.inventory
        ]
        gevent.joinall(greenlets) 

def testGevent():

    # 创建一个协程对象，创建的时候就自动运行了，不需要手动启动
    g1 = gevent.spawn(work5, 10)
    g2 = gevent.spawn(work5, 10)
    g3 = gevent.spawn(work5, 10)

    # 阻塞等待协程执行完毕
    # 没使用monkey.patch_all()破解的时候不会自动切换，破解后就会随机协程
    # g1.join()
    # g2.join()
    # g3.join()

    # 程序从上到下执行，不管之前有没有异步代码，遇到join相当于一面墙，堵住下面的路
    gevent.joinall([g1,g2,g3])

    print("全部协程结束完毕.") 

def verify_async(case_list,coroutine):
        """
        Verify used gevent lib
        :param case_list:
        :param coroutine:
        :return:
        """
        from gevent import monkey
        monkey.patch_all()
        result = []
        geventPool = pool.Pool(coroutine)
        tasks = [geventPool.spawn(Verify.request_and_verify, case) for case in case_list]
        gevent.joinall(tasks)
        for i in tasks:
            if i.value is not None:
                result.append(i.value)
        print_info('Total Verify-Case is: %s, %s error happened.' % (len(case_list), Verify.ERROR_COUNT))
        return result 

def test_two_callbacks(self):
        response_greenlet1 = gevent.spawn(self.client.send_task, 'message1')
        response_greenlet2 = gevent.spawn(self.client.send_task, 'message2')

        gevent.joinall([response_greenlet1, response_greenlet2])

        self.assertEquals('message1::response', response_greenlet1.get())
        self.assertEquals('message2::response', response_greenlet2.get())

        spans = self.tracer.finished_spans()
        self.assertEquals(len(spans), 2)

        for span in spans:
            self.assertEquals(span.tags.get(tags.SPAN_KIND, None),
                              tags.SPAN_KIND_RPC_CLIENT)

        self.assertNotSameTrace(spans[0], spans[1])
        self.assertIsNone(spans[0].parent_id)
        self.assertIsNone(spans[1].parent_id) 

def test_anonymous_with_rate_limit(client, client_ip, mocker, configs, url):
    cfg = deepcopy(configs[0])
    if '127.0.0.1' in cfg:
        cfg[client_ip] = cfg.pop('127.0.0.1')
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', cfg)

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(5):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    assert queue.get_nowait() == 429 

def test_anonymous_no_concurrent_limit_because_remain_count(
        client, client_ip, mocker, url):
    mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
        '__anonymous__': {
            'ttl': 100,
            'capacity': 100,
        }
    })

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def test_anonymous_with_concurrent_limit(
        client, client_ip, mocker, configs, url):
    cfg = deepcopy(configs[0])
    if '127.0.0.1' in cfg:
        cfg[client_ip] = cfg.pop('127.0.0.1')
    mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', cfg)

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    assert queue.get_nowait() == 429 

def denoise(filepath, media_playback_filepath):
    filepath_denoised = "/tmp/denoised.raw"
    filepath_denoised_wav = "/tmp/denoised" + ".wav"

    greenlets = [gevent.spawn(to_pcm_s16le_8khz, path) for path in filepath, media_playback_filepath]
    gevent.joinall(greenlets)

    filepath_raw, media_playback_filepath_raw = [greenlets[0].value, greenlets[1].value]

    subprocess.call(['/vagrant/core/third_party/rel643_aec_x64_demo/aecdemo', '-mic', filepath_raw,
                     '-spk', media_playback_filepath_raw, '-o', filepath_denoised])

    subprocess.call(['avconv', '-y', '-f', 's16le', '-ar', '8k', '-ac', '1', '-i', filepath_denoised, filepath_denoised_wav])

    return filepath_denoised_wav

# Use globals trick to prevent TensorFlow executor hang when executing on multiple threads. 

def _get_total_users(trending_problems,
                     start_date,
                     end_date):

    def _perform_query(problem, start_date):
        sql = """
                 SELECT COUNT(id)
                 FROM `submission`
                 WHERE ((problem_link = '%s')
                   AND  (time_stamp >= '%s'))
                 GROUP BY user_id, custom_user_id
              """ % (problem["submission"]["problem_link"],
                     start_date)
        res = db.executesql(sql)
        problem["unique"] = len(res)

    threads = []
    for problem in trending_problems:
        threads.append(gevent.spawn(_perform_query, problem, start_date))

    gevent.joinall(threads)

    return trending_problems

# ---------------------------------------------------------------------------- 

def join_all_consumer_shedual_task_thread(cls):
        nb_print(
            (cls.schedulal_thread_to_be_join, len(cls.schedulal_thread_to_be_join), '模式：', cls.global_concurrent_mode))
        if cls.schedual_task_always_use_thread:
            for t in cls.schedulal_thread_to_be_join:
                nb_print(t)
                t.join()
        else:
            if cls.global_concurrent_mode == 1:
                for t in cls.schedulal_thread_to_be_join:
                    nb_print(t)
                    t.join()
            elif cls.global_concurrent_mode == 2:
                # cls.logger.info()
                nb_print(cls.schedulal_thread_to_be_join)
                gevent.joinall(cls.schedulal_thread_to_be_join, raise_error=True, )
            elif cls.global_concurrent_mode == 3:
                for g in cls.schedulal_thread_to_be_join:
                    # eventlet.greenthread.GreenThread.
                    nb_print(g)
                    g.wait() 

def __init__(self, target=''):
        Greenlet.__init__(self)
        self.target=target
        r=requests.get(target)
        self.headers=r.headers
        self.content=r.content

        gevent.spawn(self.loadplugins).join()

        gevent.joinall([
            gevent.spawn(self.runplugin, '1'),
            gevent.spawn(self.runplugin, '2'),
            gevent.spawn(self.runplugin, '3'),
            gevent.spawn(self.runplugin, '4'),
            gevent.spawn(self.runplugin, '5'),
            gevent.spawn(self.runplugin, '6'),
            gevent.spawn(self.runplugin, '7'),
            gevent.spawn(self.runplugin, '8'),
            gevent.spawn(self.runplugin, '9'),
            gevent.spawn(self.runplugin, '10'),

        ]) 

def result_concurrency(urls):
    ''' Open all the greenlet threads '''
    in_parallel = 100
    pool = Pool(in_parallel)
    jobs = [pool.spawn(action, url) for url in urls]
    return joinall(jobs) 

def run(self):
        start = time.time()
        print("[+] Begin to brute domain")
        i = 0
        REMOVE_FLAG = False
        while not self.queues.empty() or not self.queue_sub.empty():
            i = i + 1
            if REMOVE_FLAG:
                self.filter_black_subdomain()
            domain_list = self.get_block()
            coroutines = [gevent.spawn(self.query_domain, l)
                          for l in domain_list]
            try:
                gevent.joinall(coroutines)
            except KeyboardInterrupt:
                print('user stop')
                sys.exit(1)

            self.deweighting_subdomain(REMOVE_FLAG)
            self.cmd_print(self.queues.qsize(), start, time.time(), i)

            if not REMOVE_FLAG:
                self.handle_data()
                self.raw_write_disk()

            if (self.queues.qsize() < 30000 and self.queue_sub.qsize() > 0):
                REMOVE_FLAG = True
                while (self.queues.qsize() < 200000):
                    if not self.generate_sub():
                        break

        self.handle_data()
        self.raw_write_disk()
        self.deal_write_disk()
        self.collect_cname()
        print(self.cmdline)
        print("[+] Brute over") 

def handle(self, source, address):  # pylint:disable=method-hidden
        log('%s:%s accepted', *address[:2])
        try:
            dest = create_connection(self.dest)
        except IOError as ex:
            log('%s:%s failed to connect to %s:%s: %s', address[
                0], address[1], self.dest[0], self.dest[1], ex)
            return
        forwarders = (gevent.spawn(forward, source, dest, self),
                      gevent.spawn(forward, dest, source, self))
        gevent.joinall(forwarders) 

def run(self):
        while True:
            self.proxies.clear()
            str = 'IPProxyPool----->>>>>>>>beginning'
            sys.stdout.write(str + "\r\n")
            sys.stdout.flush()
            proxylist = sqlhelper.select()

            spawns = []
            for proxy in proxylist:
                spawns.append(gevent.spawn(detect_from_db, self.myip, proxy, self.proxies))
                if len(spawns) >= MAX_CHECK_CONCURRENT_PER_PROCESS:
                    gevent.joinall(spawns)
                    spawns= []
            gevent.joinall(spawns)
            self.db_proxy_num.value = len(self.proxies)
            str = 'IPProxyPool----->>>>>>>>db exists ip:%d' % len(self.proxies)

            if len(self.proxies) < MINNUM:
                str += '\r\nIPProxyPool----->>>>>>>>now ip num < MINNUM,start crawling...'
                sys.stdout.write(str + "\r\n")
                sys.stdout.flush()
                spawns = []
                for p in parserList:
                    spawns.append(gevent.spawn(self.crawl, p))
                    if len(spawns) >= MAX_DOWNLOAD_CONCURRENT:
                        gevent.joinall(spawns)
                        spawns= []
                gevent.joinall(spawns)
            else:
                str += '\r\nIPProxyPool----->>>>>>>>now ip num meet the requirement,wait UPDATE_TIME...'
                sys.stdout.write(str + "\r\n")
                sys.stdout.flush()

            time.sleep(UPDATE_TIME) 

def process_start(tasks, myip, queue2, cntl):
    spawns = []
    for task in tasks:
        spawns.append(gevent.spawn(detect_proxy, myip, task, queue2))
    gevent.joinall(spawns)
    cntl.put(os.getpid())  # 子进程退出是加入控制队列 

def clientprocess():
    t0 = time.time()
    clients = [gevent.spawn(client) for _ in range(N_CLIENTS)]
    gevent.joinall(clients)
    duration = time.time() - t0
    print('%s clients served within %.2f s.' % (N_CLIENTS, duration)) 

def start(self, ip):
		try:
			gevents = []
			for port in self.ports:
				gevents.append(gevent.spawn(self.scan, ip, int(port)))
			gevent.joinall(gevents)
		except Exception as e:
			pass 

def refresh_editorials():
    """
        Refresh editorial links in the database
    """

    ptable = db.problem
    stable = db.submission

    today = datetime.datetime.now().strftime("%Y-%m-%d")
    before_30 = (datetime.datetime.now() - \
                 datetime.timedelta(30)).strftime("%Y-%m-%d")

    query = ((ptable.editorial_added_on == None) |
             (ptable.editorial_added_on >= before_30)) & \
            ((ptable.editorial_link == None) | \
             (ptable.editorial_link == ""))
    no_editorial = db(query).select(ptable.id)
    no_editorial = [x.id for x in no_editorial]

    today = datetime.datetime.now().strftime("%Y-%m-%d")

    threads = []
    workers = 49

    # Start retrieving tags for the problems
    # that are not in problem table
    for i in xrange(0, len(no_editorial), workers):
        threads = []
        # O God I am so smart !!
        for problem_id in no_editorial[i : i + workers]:
            threads.append(gevent.spawn(get_editorial, problem_id, today))

        gevent.joinall(threads)

    print "Total Inserted: [%d]" % (total_inserted)
    print "Total Updated: [%d]" % (total_updated)
    print "Total Not-changed: [%d]" % (not_updated) 

def run(self):
        for _ in range(1, LISTENERS_COUNT + 1):
            conn = self.broker_connection()

            listener = self.listener_fact()
            self.listeners.append(gevent.spawn(listener))
        self.log('Started {} listeners'.format(LISTENERS_COUNT))

        gevent.joinall(self.listeners) 

def get_proxies(self, quantity, type):
        '''
        quantity: 数量
        type: 类型
            1.国内高匿代理
            2.国内普通代理
            3.国外高匿代理
            4.国外普通代理
        '''
        url_queue = Queue()
        need_pages = int(math.ceil(quantity/15))
        # 判断类型
        if type == 1:
            # 国内高匿代理
            base_url = self.domestic_gn_url
        elif type == 2:
            # 国内普通代理
            base_url = self.domestic_pt_url
        elif type == 3:
            # 国外高匿代理
            base_url = self.abroad_gn_url
        elif type == 4:
            # 国外普通代理
            base_url = self.abroad_pt_url
        # 获取所需要的页面URL
        for index in range(need_pages):
            url = base_url.format(index+1)
            url_queue.put(url)
        # 处理所有URL,开启2个协程
        gevent_list = []
        gevent_list.append(
            gevent.spawn(self.fetch_urls, url_queue, quantity)
        )
        gevent.joinall(gevent_list) 

def update_graph_data(self, sites):
        threads = []
        for site in sites:
            if self.handles.has_key(site + "_handle") and self.handles[site + "_handle"] != "":
                threads.append(gevent.spawn(getattr(self,
                                                    "fetch_site_rating_history"),
                                            site))

        gevent.joinall(threads)
        if self.retrieval_failed == False:
            current.REDIS_CLIENT.delete("get_stopstalk_rating_history_" + self.user_record.stopstalk_handle)
            self.write_to_filesystem()
            self.user_record.update_record(graph_data_retrieved=True)
        else:
            log_line(self.get_debug_statement() + "Writing to file skipped") 

def test_logged_no_rate_limit_because_redis_error(
        client, client_ip, test_user, test_token, mocker, error_method):
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
        test_user.username: {
            'rate': 1,
            'capacity': 3,
        }
    })
    mocker.patch.object(redis_client, error_method, side_effect=Exception)

    def worker(queue):
        response = client.get(
            '/api/need_login', headers={
                'Authorization': test_token,
            })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait()