Python gevent.queue.Queue() Examples

def test_anonymous_no_concurrent_limit_because_remain_count(
        client, client_ip, mocker, url):
    mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
        '__anonymous__': {
            'ttl': 100,
            'capacity': 100,
        }
    })

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def __init__(self, *args, **kwargs):
        """Use :class:`~gnsq.Consumer` instead.

        .. deprecated:: 1.0.0
        """
        setattr(self, 'async', kwargs.pop('async', False))

        max_concurrency = kwargs.pop('max_concurrency', 0)

        if max_concurrency < 0:
            self.max_concurrency = cpu_count()
        else:
            self.max_concurrency = max_concurrency

        if self.max_concurrency:
            self.queue = Queue()
        else:
            self.queue = None

        super(Reader, self).__init__(*args, **kwargs) 

def test_logged_no_concurrent_limit_because_remain_count(
        client, client_ip, test_user, test_token, mocker):
    mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
        test_user.username: {
            'ttl': 100,
            'capacity': 100,
        }
    })

    def worker(queue):
        response = client.get(
            '/api/busy_with_login', headers={
                'Authorization': test_token,
            })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def main():
    print('请注意，代理数是固定的，原理是从66ip.cn上获取的ip，它是按时间排序的，所以建议一段时间获取一次，可以同步该网站的更新。')
    num = input('请输入你获取的代理数(建议500-2000)：')
    r = requests.get(
        'http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=http%3A%2F%2Fwww.66ip.cn%2F%3Fsxb%3D%26tqsl%3D10%26ports%255B%255D2%3D%26ktip%3D%26sxa%3D%26radio%3Dradio%26submit%3D%25CC%25E1%2B%2B%25C8%25A1'.format(num))
    r.encoding = 'gb2312'

    result = re.findall(r'\d+[.]\d+[.]\d+[.]\d+[:]\d+', r.text)

    gevent_pool = gevent.pool.Pool(len(result))
    print(len(result))
    queue = Queue()
    for proxy in result:
        gevent_pool.apply_async(validate, (proxy, queue))
    gevent_pool.join()

    useful_proxies = []
    queue.put(StopIteration)
    with open('免费可用.txt', 'w') as f:
        for item in queue:
            useful_proxies.append(item)
            f.write(item + '\n')
    print('通过数量:', len(useful_proxies))
    print('通过率:', str(len(useful_proxies)/len(result)*100) + '%')
    return useful_proxies 

def __init__(self, app):
        self.config = app.config
        self.db = app.services.db
        assert self.db is not None
        super(ChainService, self).__init__(app)
        log.info('initializing chain')
        coinbase = app.services.accounts.coinbase
        self.chain = Chain(self.db, new_head_cb=self._on_new_head, coinbase=coinbase)
        log.info('chain at', number=self.chain.head.number)
        self.synchronizer = Synchronizer(self, force_sync=None)

        self.block_queue = Queue(maxsize=self.block_queue_size)
        self.transaction_queue = Queue(maxsize=self.transaction_queue_size)
        self.add_blocks_lock = False
        self.add_transaction_lock = gevent.lock.Semaphore()
        self.broadcast_filter = DuplicatesFilter()
        self.on_new_head_cbs = []
        self.on_new_head_candidate_cbs = [] 

def __init__(self, tree_hub, from_application_name=None,
                 from_cluster_name=None, with_initial=False,
                 life_span=None, metrics_tag_from=None):
        self.hub = tree_hub

        # The optional route context
        self.from_application_name = from_application_name
        self.from_cluster_name = from_cluster_name

        self.with_initial = with_initial
        self.queue = Queue()
        self.holders = set()
        self.cluster_maps = collections.defaultdict(ClusterMap)
        self.cluster_whitelist = collections.defaultdict(set)
        self.watch_map = collections.defaultdict(set)
        self.life_span = life_span
        self._metrics_tag_from = metrics_tag_from 

def test_logged_no_rate_limit_because_remain_count(
        client, client_ip, test_user, test_token, mocker):
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
        test_user.username: {
            'rate': 100,
            'capacity': 300,
        }
    })

    def worker(queue):
        response = client.get(
            '/api/need_login', headers={
                'Authorization': test_token,
            })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def test_anonymous_no_rate_limit_because_remain_count(
        client, client_ip, mocker, url):
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
        '__anonymous__': {
            'rate': 100,
            'capacity': 300,
        }
    })

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def _concurrent_execute(self, context, start_req, parser, pool, pool_size):
        queue = Queue()  # 任务队列

        # 将初始化请求加入任务队列
        for r in start_req:
            queue.put_nowait(r)

        if pool is None:
            pool = GeventPool(pool_size)

        greenlets = []

        while True:
            try:
                req = self._check_req(queue.get(timeout=1))
                if req.parser is None:
                    req.parser = parser
                greenlets.append(pool.spawn(req, context, queue))
            except Empty:
                break

        return [greenlet.get() for greenlet in greenlets] 

def test_anonymous_with_rate_limit(client, client_ip, mocker, configs, url):
    cfg = deepcopy(configs[0])
    if '127.0.0.1' in cfg:
        cfg[client_ip] = cfg.pop('127.0.0.1')
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', cfg)

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(5):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    assert queue.get_nowait() == 429 

def test_messages(self, test_uuids):
        user = User("test_user")
        connection = Connection("test_user", conn_id=test_uuids[1])
        connection.queue = Queue()
        connection2 = Connection("test_user", conn_id=test_uuids[2])
        connection2.queue = Queue()
        user.add_connection(connection)
        user.add_connection(connection2)
        user.add_message(
            {
                "type": "message",
                "no_history": False,
                "pm_users": [],
                "exclude_users": [],
            }
        )
        assert len(user.connections) == 2
        assert len(user.connections[0].queue.get()) == 1
        assert len(user.connections[1].queue.get()) == 1 

def _run_queue_item(self, item):
        name, args, kwargs = item
        if __debug__:
            self.logger.debug("Queue running {} with args '{}' kwargs '{}'"
                              .format(name, args, kwargs))
        try:
            func = getattr(self, name, None)
            if func is None:
                raise NotImplementedError(
                    "Item {} has been enqueued that has no valid function!"
                    .format(name))
            func(*args, **kwargs)
        except self.queue_exceptions as e:
            self.logger.error("Unable to process queue item, retrying! "
                              "{} Name: {}; Args: {}; Kwargs: {};"
                              .format(e, name, args, kwargs))
            return "retry"
        except Exception:
            # Log any unexpected problem, but don't retry because we might
            # end up endlessly retrying with same failure
            self.logger.error("Unkown error, queue data discarded!"
                              "Name: {}; Args: {}; Kwargs: {};"
                              .format(name, args, kwargs), exc_info=True) 

def test_default_anonymous_no_rate_limit(
        client, client_ip, switch_on, mocker, url):
    def fake_switch(name, default=True):
        if name == SWITCH_ENABLE_RATE_LIMITER:
            return switch_on
        return default

    mocker.patch.object(switch, 'is_switched_on', fake_switch)
    if not switch_on:
        mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
            '__default__': {
                'rate': 1,
                'capacity': 3,
            }
        })

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(True)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def test_logged_no_rate_limit_because_redis_error(
        client, client_ip, test_user, test_token, mocker, error_method):
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
        test_user.username: {
            'rate': 1,
            'capacity': 3,
        }
    })
    mocker.patch.object(redis_client, error_method, side_effect=Exception)

    def worker(queue):
        response = client.get(
            '/api/need_login', headers={
                'Authorization': test_token,
            })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def test_logged_with_rate_limit(
        client, client_ip, mocker, test_user, test_token,
        configs, use_username):
    if '127.0.0.1' in configs:
        configs[client_ip] = configs.pop('127.0.0.1')
    if use_username:
        configs.update({
            test_user.username: {
                'rate': 1,
                'capacity': 3,
            }
        })
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', configs)

    def worker(queue):
        response = client.get('/api/need_login', headers={
            'Authorization': test_token,
        })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(10):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    assert queue.get_nowait() == 429 

def _flush_queue(self, exit_exc=None, caller=None):
        sleep(1)
        self.logger.info("Flushing a queue of size {}"
                         .format(self.queue.qsize()))
        self.queue.put(StopIteration)
        for item in self.queue:
            self._run_queue_item(item)
        self.logger.info("Queue flush complete, Exit.") 

def test_logged_concurrent_limit_with_redis_error(
        client, client_ip, test_user, test_token, mocker,
        error_method, cause_limit):
    mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
        test_user.username: {
            'ttl': 100,
            'capacity': 1,
        }
    })
    mocker.patch.object(redis_client, error_method, side_effect=Exception)

    def worker(queue):
        response = client.get(
            '/api/busy_with_login', headers={
                'Authorization': test_token,
            })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    if cause_limit:
        assert queue.get_nowait() == 429
    else:
        with raises(Empty):
            queue.get_nowait() 

def test_login_with_concurrent_limit(
        client, client_ip, mocker, test_user, test_token,
        configs, use_username):
    if '127.0.0.1' in configs:
        configs[client_ip] = configs.pop('127.0.0.1')
    if use_username:
        configs.update({
            test_user.username: {
                'ttl': 100,
                'capacity': 1,
            }
        })
    mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', configs)

    def worker(queue):
        response = client.get('/api/busy_with_login', headers={
            'Authorization': test_token,
        })
        if response.status_code != 200:
            queue.put(response.status_code)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    assert queue.get_nowait() == 429 

def __init__(self, sock, address, id, server, config, logger, reporter):
        self.logger = logger
        self.sock = sock
        self.server = server
        self.config = config
        self.reporter = reporter

        # Seconds before sending keepalive probes
        sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 120)
        # Interval in seconds between keepalive probes
        sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 1)
        # Failed keepalive probles before declaring other end dead
        sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 5)

        self._disconnected = False
        self._authenticated = False
        self._client_state = None
        self._authed = {}
        self._client_version = None
        self._connection_time = time()
        self._id = id

        # where we put all the messages that need to go out
        self.write_queue = Queue()
        self.fp = None
        self._stopped = False 

def _start_queue(self):
        self.queue = Queue() 

def test_default_anonymous_no_concurrent_limit(
        client, client_ip, switch_on, mocker, url):
    def fake_switch(name, default=True):
        if name == SWITCH_ENABLE_CONCURRENT_LIMITER:
            return switch_on
        return default

    mocker.patch.object(switch, 'is_switched_on', fake_switch)
    if not switch_on:
        mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
            '__default__': {
                'ttl': 100,
                'capacity': 1,
            }
        })

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def __init__(self, config):
        self._configure(config)
        super(CeleryReporter, self).__init__()

        # setup our celery agent and monkey patch
        from celery import Celery
        self.celery = Celery()
        self.celery.conf.update(self.config['celery'])

        self.queue = Queue()
        self._aggr_shares = {} 

def _new_queue(*args, **kwargs):
    from six.moves.queue import Queue
    return Queue(*args, **kwargs) 

def _new_gevent_queue(*args, **kwargs):
    from gevent.queue import Queue
    return Queue(*args, **kwargs) 

def _new_queue(*args, **kwargs):
    from six.moves.queue import Queue
    return Queue(*args, **kwargs) 

def _new_gevent_queue(*args, **kwargs):
    from gevent.queue import Queue
    return Queue(*args, **kwargs) 

def __init__(self,connectionstring,poolsize,modulename='pyodbc'):
        self.conns = [DBConnection_(socket_.socketpair()) for x in xrange(poolsize)]
        self.threads = [threading.Thread(target=self.worker, args=(self.conns[x],)) for x in xrange(poolsize)]
        self.queue = queue.Queue(poolsize)
        for i in xrange(poolsize):
            self.threads[i].daemon = True
            self.threads[i].start()
            self.conns[i].connect(connectionstring,modulename)
            self.queue.put(self.conns[i])
        if KEEPALIVE_PERIOD > 0:
            self.monitor = gevent.spawn(self.keepalive) 

def subscribe():
    def gen():
        q = Queue()
        subscriptions.append(q)
        try:
            while True:
                result = q.get()
                id = result['id']
                ev = ServerSentEvent(result, id)
                yield ev.encode()
        except GeneratorExit:  # Or maybe use flask signals
            subscriptions.remove(q)
    return Response(gen(), mimetype="text/event-stream") 

def __init__(self, syncback_id, process_number, total_processes, poll_interval=1,
                 retry_interval=30, num_workers=NUM_PARALLEL_ACCOUNTS,
                 batch_size=10):
        self.process_number = process_number
        self.total_processes = total_processes
        self.poll_interval = poll_interval
        self.retry_interval = retry_interval
        self.batch_size = batch_size
        self.keep_running = True
        self.workers = gevent.pool.Group()
        # Dictionary account_id -> semaphore to serialize action syncback for
        # any particular account.
        # TODO(emfree): We really only need to serialize actions that operate
        # on any given object. But IMAP actions are already effectively
        # serialized by using an IMAP connection pool of size 1, so it doesn't
        # matter too much.
        self.account_semaphores = defaultdict(lambda: BoundedSemaphore(1))
        # This SyncbackService performs syncback for only and all the accounts
        # on shards it is reponsible for; shards are divided up between
        # running SyncbackServices.
        self.log = logger.new(component='syncback')
        syncback_assignments = config.get("SYNCBACK_ASSIGNMENTS", {})
        if syncback_id in syncback_assignments:
            self.keys = [key for key in engine_manager.engines
                         if key in syncback_assignments[syncback_id] and
                         key % total_processes == process_number]
        else:
            self.log.warn("No shards assigned to syncback server",
                          syncback_id=syncback_id)
            self.keys = []

        self.log = logger.new(component='syncback')
        self.num_workers = num_workers
        self.num_idle_workers = 0
        self.worker_did_finish = gevent.event.Event()
        self.worker_did_finish.clear()
        self.task_queue = Queue()
        self.running_action_ids = set()
        gevent.Greenlet.__init__(self) 

def __init__(self, account_id, num_connections, readonly):
        log.info('Creating Crispin connection pool',
                 account_id=account_id, num_connections=num_connections)
        self.account_id = account_id
        self.readonly = readonly
        self._queue = Queue(num_connections, items=num_connections * [None])
        self._sem = BoundedSemaphore(num_connections)
        self._set_account_info()