Python gevent.pool() Examples

def execute(self, context, start_req, parser=_default_parser,
                pool=None, pool_size=None):
        """
        :param context: 上下文对象
        :param start_req: 起始请求列表
        :param parser: Response对象解析器
        :param concurrent: 是否采用并发方式抓取
        :param pool: 指定已有的gevent pool
        """

        if pool or pool_size:
            # 并发请求
            return self._concurrent_execute(
                context, start_req, parser,
                pool, pool_size)
        else:
            # 同步请求
            return self._sync_execute(context, start_req, parser) 

def task(pool_number, types=types, tags_dict=tags_dict, sorts=sorts):
    video_douban_ids = set(get_video_douban_ids())
    global video_douban_ids

    pool = Pool(pool_number)

    for type in types:
        for tag in tags_dict[type]:
            for sort in sorts:
                pool.spawn(
                    create_requests_and_save_datas,
                    type=type,
                    tag=tag,
                    sort=sort
                )
    pool.join()

    return list(video_douban_ids) 

def _load_dns_servers(self):
        print '[+] Validate DNS servers ...'
        self.dns_servers = []
        pool = Pool(30)
        for server in open('dict/dns_servers.txt').xreadlines():
            server = server.strip()
            if server:
                pool.apply_async(self._test_server, (server,))
        pool.join()

        self.dns_count = len(self.dns_servers)
        sys.stdout.write('\n')
        print '[+] Found %s available DNS Servers in total' % self.dns_count
        if self.dns_count == 0:
            print '[ERROR] No DNS Servers available.'
            sys.exit(-1) 

def record_proxy_state(self, proxy, state):
        if not proxy: return
        proxy_url = proxy['http']
        if state == self.FAILED:
            if proxy_url in self._proxy_failed:
                self._proxy_failed[proxy_url] += 1
                if self._proxy_failed[proxy_url] > self._proxy_failed_threshold:
                    try:
                        print '!!! remove proxy: %s , left: [%s] !!!' % (
                            proxy_url,
                            len(self._pool)
                        )
                        # TODO proxy pool changed, changes should also be made to recording data structures such as _host_proxy_time
                        self._pool.remove(proxy)
                    except:
                        pass
            else:
                self._proxy_failed[proxy_url] = 1
        elif state == self.SUCCESS:
            if proxy_url in self._proxy_failed:
                if self._proxy_failed[proxy_url] > 0:
                    self._proxy_failed[proxy_url] -= 1
        else:
            print '!!!!! invalid proxy state: %s !!!!' % state 

def add(self, greenlet):
        source = greenlet.args[2][1][0] + ':' + str(greenlet.args[2][1][1])
        socket = greenlet.args[2][0]

        # With 1, we avoid the wait caused by the semaphore
        if self.free_count() < 2:
            # /!\ pool full, untracking oldest greenlet /!\
            oldest_source = self.open_connection[0]
            oldest_greenlet = self.open_connection_dico_ip[oldest_source]

            #kill the greenlet, this also close its associated socket
            self.killone(oldest_greenlet, block=False)

        #Add the connection to the dicos
        self.open_connection.append(source)
        self.open_connection_dico_ip[source] = greenlet
        self.open_connection_dico_green[str(greenlet)] = source
        gevent.pool.Pool.add( self, greenlet)

    # discard the greenlet, free one slot of the pool 

def _concurrent_execute(self, context, start_req, parser, pool, pool_size):
        queue = Queue()  # 任务队列

        # 将初始化请求加入任务队列
        for r in start_req:
            queue.put_nowait(r)

        if pool is None:
            pool = GeventPool(pool_size)

        greenlets = []

        while True:
            try:
                req = self._check_req(queue.get(timeout=1))
                if req.parser is None:
                    req.parser = parser
                greenlets.append(pool.spawn(req, context, queue))
            except Empty:
                break

        return [greenlet.get() for greenlet in greenlets] 

def main(version="1.0.0"):
    conf = Configuration(description="", version=version)
    conf.register_str("log_level", default="INFO",
                      help="The level of the log, such as debug, info, etc.")
    conf.register_str("log_file", default="", help="The file path of the log.")
    conf.register_int("thread_num", default=0, help="The size of the coroutine pool.")
    conf.parse()

    if conf.thread_num > 0:
        global taskpool, spawn
        taskpool = gevent.pool.Pool(size=conf.thread_num)
        spawn = taskpool.spawn

    init_logging(LOG, conf.log_level, conf.log_file)

    # TODO:) 

def spawn_list(self):
        """"
        The spawn() method begins a new greenlet with the given arguments
        (which are passed to the greenlet constructor) and adds it to the
        collection of greenlets this group is monitoring.

        We return a list of the newly started greenlets, used in a later
        'joinall` call.

        :return: A list of the newly started greenlets.
        """

        # here, workgroups is a list of Type[BaseGroup] objects
        workgroups = [val for val in self.workgroups.values()]
        spawn_list = [self.pool.spawn(self.monitor, worker) for work_group in
                      workgroups for worker in work_group]

        # we get a blocking error if we spawn the manager first, so spawn it last
        spawn_list.append(self.pool.spawn(self.manage))

        return spawn_list 

def run(self):
		# 先判断结果是否存在过
		# 以及报告目录是否存在
		findreport(self.document)

		# 先检测一个不存在的地址
		# 如果能解析，则证明做了一些保护措施
		# 将对方地址记录下来，作为block的标准
		block_check_results = self.checkdomain('d312379196bd822558ca7dfb3c95ba61.'+self.options['target'],'block')
		
		if block_check_results:
			self.blockip = block_check_results[0]

		# 构建字典
		dic_list = (dic.strip('\n')+'.'+self.options['target'] for dic in open(getpath() + '/' +self.options['dictname'],'r'))
		# 协程爆破测试
		self.pool.map(self.checkdomain,dic_list) 

def init_process(self):
        pass
        # pool = Pool(self.process_number)
        # return pool 

def _discard(self, greenlet):
        to_del_greenlet = str(greenlet)
        to_del_source = self.open_connection_dico_green[to_del_greenlet]
        gevent.pool.Pool._discard(self, greenlet)

        #cleaning dicos
        del self.open_connection_dico_ip[to_del_source]
        del self.open_connection_dico_green[to_del_greenlet]
        self.open_connection.remove(to_del_source) 

def __init__(self, common_gap=15, proxies_file=''):
        self._proxy_file = proxies_file
        self._pool = []  # list of proxy dicts loading from proxy_file
        self.load(proxies_file)
        self._common_gap = common_gap
        self._special_gap = {} # {host:gap}
        self._host_proxy_time = {} # {host: [[index_of_pool, last_use_time] * len(self._pool)]}, `pool_index` indicates the proxy in the pool,  `last_use_time` record the time accessing `host`
        self._host_last_proxy_id = {} # {host: last_proxy_index}, `last_proxy_index` indicates the proxies in the pool used to access `host`s last time.
        self._proxy_failed = {} # {proxy: failed_count}
        self._proxy_failed_threshold = 7
        self.FAILED = 0
        self.SUCCESS = 1

        # for no proxy
        self._host_last_time = {} # {host: last_access_time} 

def __init__(self,options):
		self.options = options
		self.blockip = None
		self.keywords = False
		self.pool = Pool(self.options['threads_count'])
		self.document = self.options['target'].replace('.','_')+'.txt'
		socket.setdefaulttimeout(self.options['timeout'])

	# 域名解析 

def coroutine_response(self, size=None, stream=False, exception_handler=None, status_only=False):
        g_requests = list(self.__grequests)

        pool = Pool(size) if size else None
        jobs = [send(r, pool, stream=stream) for r in g_requests]
        gevent.joinall(jobs)

        ret = []
        if status_only:
            for request in g_requests:
                if request.response:
                    ret.append(copy.copy(True))
                else:
                    ret.append(copy.copy(False))
            return ret

        for request in g_requests:
            if request.response:
                ret.append(request.response)
            elif exception_handler:
                exception_handler(request, request.exception)



        return ret 

def __len__(self):
        """make sure that pool always evaluates to true."""
        l = gevent.pool.Pool.__len__(self)
        if not l:
            return 1
        return l 

def main(domain, masscanOutput, urlOutput):
    domainsFound = {}
    domainsNotFound = {}
    if (not masscanOutput and not urlOutput):
        print("[+]: Downloading domain list...")
    response = collectResponse(domain)
    if (not masscanOutput and not urlOutput):
        print("[+]: Download of domain list complete.")
    domains = collectDomains(response)
    if (not masscanOutput and not urlOutput):
        print("[+]: Parsed %s domain(s) from list." % len(domains))
    
    pool = Pool(15)
    greenlets = [pool.spawn(resolve, domain) for domain in domains]
    pool.join(timeout = 1)
    for greenlet in greenlets:
        result=greenlet.value
        if (result):
            for ip in result.values():
                if ip is not 'none':
                    domainsFound.update(result)
                else:
                    domainsNotFound.update(result)

    if (urlOutput):
        printUrls(sorted(domains))
    if (masscanOutput):
        printMasscan(domainsFound)
    if (not masscanOutput and not urlOutput):
        print("\n[+]: Domains found:")
        printDomains(domainsFound)
        print("\n[+]: Domains with no DNS record:")
        printDomains(domainsNotFound) 

def run_items(self, items, session, workers=None):
        import gevent
        import gevent.monkey
        import gevent.pool
        gevent.monkey.patch_all()
        pool = gevent.pool.Pool(size=workers)
        for index, item in enumerate(items):
            pool.spawn(self._run_next_item, session, item, index)
        pool.join() 

def command_recover(self, event, duration, pool=4, mode=None):
        if mode == 'global':
            channels = list(self.state.channels.values())
        else:
            channels = list(event.guild.channels.values())

        start_at = parse_duration(duration, negative=True)

        pool = Pool(pool)

        total = len(channels)
        msg = event.msg.reply('Recovery Status: 0/{}'.format(total))
        recoveries = []

        def updater():
            last = len(recoveries)

            while True:
                if last != len(recoveries):
                    last = len(recoveries)
                    msg.edit('Recovery Status: {}/{}'.format(len(recoveries), total))
                gevent.sleep(5)

        u = self.spawn(updater)

        try:
            for channel in channels:
                pool.wait_available()
                r = Recovery(self.log, channel, start_at)
                pool.spawn(r.run)
                recoveries.append(r)
        finally:
            pool.join()
            u.kill()

        msg.edit('RECOVERY COMPLETED ({} total messages)'.format(
            sum([i._recovered for i in recoveries])
        )) 

def send(r, pool=None, stream=False):
    if pool != None:
        return pool.spawn(r.send, stream=stream)
    return gevent.spawn(r.send, stream=stream) 

def run_gevent(self):
        # 程序如果意外退出，协程部分不会被kill掉，需要下面这一行代码来关联信号量
        # gevent.signal(signal.SIGQUIT, gevent.shutdown)
        # 构建一个和账号数量相同的协程池
        coroutine_pool = gevent.pool.Pool(len(self._accounts))
        self.__sessions = [requests.session() for _ in range(len(self.accounts))]
        self.__user_ids = [None for _ in range(len(self.accounts))]
        self.__buy_status = [0 for _ in range(len(self.accounts))]
        
        # 传入账号的序号和账号
        for account_id, account in enumerate(self._accounts):
            coroutine_pool.apply_async(self.do, (account_id, account, self.sizes[account_id], self.proxies[account_id]))
        coroutine_pool.join() 

def alter_address(self, payload):
        coroutine_pool = gevent.pool.Pool()
        for account_id in range(len(self._accounts)):
            coroutine_pool.apply_async(self.alter_address_by_session,
                                       (account_id, self.__sessions[account_id], self.__user_ids[account_id], payload, self.proxies[account_id]))
        coroutine_pool.join() 

def check(self):
        pool = gevent.pool.Pool(len(self.proxies))
        for index, ip in enumerate(self.proxies):
            pool.apply_async(self.ip_delay, (index, ip))
        pool.join()
        self.trigger.emit([0]) 

def run(self) -> None:
        """
        Starts all poller instances.
        After that, polling is performed in regular intervals specified by the polling_delay_ms property.
        By default this function blocks the current thread.
        """
        self._validation_service.validate_all_addresses()

        self._logger.info("Gateway Application started")

        task_group = gevent.pool.Group()

        gevent.signal(signal.SIGINT, self._coin_transaction_polling_service.cancel)
        gevent.signal(signal.SIGINT, self._waves_transaction_polling_service.cancel)

        task_group.start(self._coin_transaction_polling_service)
        task_group.start(self._waves_transaction_polling_service)

        attempt_list_workers = self._create_attempt_list_workers()

        for worker in attempt_list_workers:
            gevent.signal(signal.SIGINT, worker.cancel)
            task_group.start(worker)

        http = gevent.pywsgi.WSGIServer(
            (self._host, self._port), self._flask, log=gevent.pywsgi.LoggingLogAdapter(self._logger.getChild('pywsgi')))

        gevent.signal(signal.SIGINT, http.close)

        self._logger.info('Listening on %s:%s', self._host, str(self._port))

        http.serve_forever()

        task_group.join(raise_error=True) 

def _filter_transactions(self, transactions: List[Transaction]) -> List[Transaction]:
        """Filters all transactions in parallel. The results may not have the same order."""
        filter_task_group = pool.Group()
        filter_task_results = filter_task_group.imap_unordered(self._filter_transaction_task, transactions)
        return filter_array(lambda el: el is not None, filter_task_results) 

def run(args):
    if args.download:
        resolvers = download_resolvers()
    else:
        resolvers = load_resolvers(args.resolvers)
    random.shuffle(resolvers)

    pool = gevent.pool.Pool(args.concurrency)

    bar = progressbar.ProgressBar(redirect_stdout=True, redirect_stderr=True)
    for resolver in bar(resolvers):
        pool.add(gevent.spawn(check_resolver, args, resolver))
    pool.join() 

def __init__(self, pool_size, retry=3, selenium_cookies=None):
        self.pool = Pool(pool_size)
        self.session = self._get_http_session(pool_size, pool_size, retry, selenium_cookies)
        self.retry = retry
        self.dir = ''
        self.succed = {}
        self.failed = []
        self.ts_total = 0
        self._result_file_name = None 

def _download(self, ts_list):
        self.pool.map(self._worker, ts_list)
        if self.failed:
            ts_list = self.failed
            self.failed = []
            self._download(ts_list) 

def __init__(self, qemu, pbw, runner, persist_dir=None, block_private_addresses=False):
        self.group = gevent.pool.Group()
        self.queue = gevent.queue.Queue()
        self.qemu = qemu
        self.pbw = pbw
        self.runner = runner
        self.runtime_id = JSRuntime.runtimeCount
        self.persist_dir = persist_dir
        self.block_private_addresses = block_private_addresses
        JSRuntime.runtimeCount += 1 

def create_server(self, sock, pool_size):
        if pool_size:
            pool = gevent.pool.Pool(pool_size)
        else:
            pool = "default"
        self.instance = GeventResultServerWorker(sock, spawn=pool)
        self.instance.do_run() 

def dequeue_job_and_maintain_ttl(self, timeout):
        if self._stop_requested:
            raise StopRequested()

        result = None
        while True:
            if self._stop_requested:
                raise StopRequested()

            self.heartbeat()

            if self.gevent_pool.full():
                self.set_state(WorkerStatus.BUSY)
                self.log.warning(
                    "RQ GEVENT worker greenlet pool empty current size %s",
                    self.gevent_pool.size,
                )

            while self.gevent_pool.full():
                gevent.sleep(0.1)
                if self._stop_requested:
                    raise StopRequested()

            try:
                result = self.queue_class.dequeue_any(
                    self.queues, timeout, connection=self.connection
                )
                self.set_state(WorkerStatus.IDLE)
                if result is not None:
                    job, queue = result
                    self.log.info(
                        "%s: %s (%s)"
                        % (green(queue.name), blue(job.description), job.id)
                    )
                break
            except DequeueTimeout:
                pass

        self.heartbeat()
        return result