Python gevent.queue() Examples

def fetch_urls(self, queue, quantity):
        while not queue.empty():
            url = queue.get()
            html = self.s.get(url, headers=self.headers).text
            pq = PyQuery(html)
            size = pq.find('tbody tr').size()
            for index in range(size):
                item = pq.find('tbody tr').eq(index)
                ip = item.find('td').eq(0).text()
                port = item.find('td').eq(1).text()
                _type = item.find('td').eq(3).text()
                self.result_arr.append({
                    str(_type).lower(): '{0}://{1}:{2}'.format(str(_type).lower(), ip, port)
                })
                if len(self.result_arr) >= quantity:
                    break 

def test_anonymous_with_concurrent_limit(
        client, client_ip, mocker, configs, url):
    cfg = deepcopy(configs[0])
    if '127.0.0.1' in cfg:
        cfg[client_ip] = cfg.pop('127.0.0.1')
    mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', cfg)

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    assert queue.get_nowait() == 429 

def __init__(self, device, prefix, watermark, attributes, persistent):
        super(DevicePusher, self).__init__()

        self.device = device
        self.xapi = pan.xapi.PanXapi(
            tag=self.device.get('tag', None),
            api_username=self.device.get('api_username', None),
            api_password=self.device.get('api_password', None),
            api_key=self.device.get('api_key', None),
            port=self.device.get('port', None),
            hostname=self.device.get('hostname', None),
            serial=self.device.get('serial', None)
        )

        self.valid_device_version = None
        self.prefix = prefix
        self.attributes = attributes
        self.watermark = watermark
        self.persistent = persistent

        self.q = gevent.queue.Queue() 

def test_anonymous_no_concurrent_limit_because_remain_count(
        client, client_ip, mocker, url):
    mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
        '__anonymous__': {
            'ttl': 100,
            'capacity': 100,
        }
    })

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def usecase_child_d(forthreader, backwriter):
    recvqueue = gevent.queue.Queue()
    def g_from_forthpipe_to_q(forthreader):
        while True:
            m = forthreader.get()
            recvqueue.put(m)
            if m == "STOP":
                break

    def g_from_q_to_backpipe(backwriter):
        while True:
            m = recvqueue.get()
            backwriter.put(m)
            if m == "STOP":
                break

    g1 = gevent.spawn(g_from_forthpipe_to_q, forthreader)
    g2 = gevent.spawn(g_from_q_to_backpipe, backwriter)
    g1.get()
    g2.get() 

def test_anonymous_with_rate_limit(client, client_ip, mocker, configs, url):
    cfg = deepcopy(configs[0])
    if '127.0.0.1' in cfg:
        cfg[client_ip] = cfg.pop('127.0.0.1')
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', cfg)

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(5):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    assert queue.get_nowait() == 429 

def test_logged_no_rate_limit_because_remain_count(
        client, client_ip, test_user, test_token, mocker):
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
        test_user.username: {
            'rate': 100,
            'capacity': 300,
        }
    })

    def worker(queue):
        response = client.get(
            '/api/need_login', headers={
                'Authorization': test_token,
            })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def make_channel(self):
    """Returns a single-slot communication device for passing data and control
    between concurrent functions.

    This is useful for running 'background helper' type concurrent processes.

    NOTE: It is strongly discouraged to pass Channel objects outside of a recipe
    module. Access to the channel should be mediated via
    a class/contextmanager/function which you return to the caller, and the
    caller can call in a makes-sense-for-your-moudle's-API way.

    See ./tests/background_helper.py for an example of how to use a Channel
    correctly.

    It is VERY RARE to need to use a Channel. You should avoid using this unless
    you carefully consider and avoid the possibility of introducing deadlocks.

    Channels will raise ValueError if used with @@@annotation@@@ mode.
    """
    if not self.concurrency_client.supports_concurrency: # pragma: no cover
      # test mode always supports concurrency, hence the nocover
      raise ValueError('Channels are not allowed in @@@annotation@@@ mode')
    return gevent.queue.Channel() 

def __init__(self, device, prefix, watermark, attributes, persistent):
        super(DevicePusher, self).__init__()

        self.device = device
        self.xapi = pan.xapi.PanXapi(
            tag=self.device.get('tag', None),
            api_username=self.device.get('api_username', None),
            api_password=self.device.get('api_password', None),
            api_key=self.device.get('api_key', None),
            port=self.device.get('port', None),
            hostname=self.device.get('hostname', None),
            serial=self.device.get('serial', None)
        )

        self.prefix = prefix
        self.attributes = attributes
        self.watermark = watermark
        self.persistent = persistent

        self.q = gevent.queue.Queue() 

def test_request_data_include_useless_keys(long_poll, test_application_name):
    queue = long_poll(custom_payload={
        'life_span': 0,
        'service': {
            test_application_name: ['stable']
        },
        'trigger': 1
    })

    event = queue.get(timeout=5)
    assert event['message'] == 'all'
    assert event['body'] == {
        'service': {test_application_name: {'stable': {}}},
        'switch': {},
        'config': {},
        'service_info': {}
    }

    assert queue.empty() 

def __init__(self, name, chassis, config):
        self.table = None
        self.agg_table = None

        self._actor_queue = gevent.queue.Queue(maxsize=128)
        self._actor_glet = None
        self._actor_commands_ts = collections.defaultdict(int)
        self._poll_glet = None
        self._age_out_glet = None
        self._emit_counter = 0

        self.last_run = None
        self.last_successful_run = None
        self.last_ageout_run = None
        self._sub_state = None
        self._sub_state_message = None

        self.poll_event = gevent.event.Event()

        self.state_lock = RWLock()

        super(BasePollerFT, self).__init__(name, chassis, config) 

def test_delete_config(zk, test_application_name, long_poll):
    zk.create(
        '/huskar/config/%s/alpha/DB_URL' % test_application_name, 'mysql://',
        makepath=True)
    queue = long_poll()

    event = queue.get(timeout=5)
    assert event['message'] == 'all'
    assert event['body'] == {
        'service': {test_application_name: {'stable': {}, 'foo': {}}},
        'switch': {test_application_name: {'stable': {}}},
        'config': {
            test_application_name: {'alpha': {
                'DB_URL': {u'value': u'mysql://'},
            }},
        },
        'service_info': {}
    }

    zk.delete('/huskar/config/%s/alpha/DB_URL' % test_application_name)
    event = queue.get(timeout=5)
    assert event['message'] == 'delete'
    assert event['body'] == {'config': {
        test_application_name: {'alpha': {'DB_URL': {u'value': None}}}
    }} 

def __init__(self, id, parser, driver, cpu=None, is_process=True):
        self.parser = parser
        self.driver = driver
        self.id = id

        self._running = False
        self._rpc_task = None
        self._events_task = None
        self._health_task = None

        self.queue = gevent.queue.Queue(maxsize=10000)
        self.cpu = cpu
        self.is_process = is_process

        self.logger = init_logging("main.{}".format(self.id))

        self.error_mr = MetricsRecorder("sniffer.main.error")
        self.msg_mr = MetricsRecorder("sniffer.main.msg")
        self.event_mr = MetricsRecorder("sniffer.main.event")
        self.rpc_mr = MetricsRecorder("sniffer.main.rpc")
        self.main_mr = MetricsRecorder("sniffer.main.loop")

        self.urltree = URLTree() 

def __init__(self, app):
        log.debug('initializing JSONRPCServer')
        BaseService.__init__(self, app)
        self.app = app

        self.dispatcher = LoggingDispatcher()
        # register sub dispatchers
        for subdispatcher in self.subdispatcher_classes():
            subdispatcher.register(self)

        transport = WsgiServerTransport(queue_class=gevent.queue.Queue)
        # start wsgi server as a background-greenlet
        self.listen_port = app.config['jsonrpc']['listen_port']
        self.listen_host = app.config['jsonrpc']['listen_host']
        self.wsgi_server = gevent.wsgi.WSGIServer((self.listen_host, self.listen_port),
                                                  transport.handle, log=WSGIServerLogger)
        self.rpc_server = RPCServerGreenlets(
            transport,
            JSONRPCProtocol(),
            self.dispatcher
        )
        self.default_block = 'latest' 

def test_create_config(zk, test_application_name, long_poll):
    queue = long_poll()

    event = queue.get(timeout=5)
    assert event['message'] == 'all'
    assert event['body'] == {
        'service': {test_application_name: {'stable': {}, 'foo': {}}},
        'switch': {test_application_name: {'stable': {}}},
        'config': {test_application_name: {}},
        'service_info': {},
    }
    assert queue.empty()

    zk.create(
        '/huskar/config/%s/alpha/DB_URL' % test_application_name, 'mysql://',
        makepath=True)
    event = queue.get(timeout=5)
    assert event['message'] == 'update'
    assert event['body'] == {'config': {
        test_application_name: {'alpha': {'DB_URL': {u'value': u'mysql://'}}},
    }} 

def _concurrent_execute(self, context, start_req, parser, pool, pool_size):
        queue = Queue()  # 任务队列

        # 将初始化请求加入任务队列
        for r in start_req:
            queue.put_nowait(r)

        if pool is None:
            pool = GeventPool(pool_size)

        greenlets = []

        while True:
            try:
                req = self._check_req(queue.get(timeout=1))
                if req.parser is None:
                    req.parser = parser
                greenlets.append(pool.spawn(req, context, queue))
            except Empty:
                break

        return [greenlet.get() for greenlet in greenlets] 

def manage(self):
        """"
        Manage will hand out work when the appropriate Worker is free.
        The manager timeout must be less than worker timeout, or else, the
        workers will be idled and shutdown.
        """
        try:
            while True:
                for name, workgroup in self.workgroups.items():
                    for qname, q in self.qitems.items():
                        if name == qname: # workgroup name must match tracker name
                            # a tracker with the same name as workgroup name, is...
                            # ...effectively, the workgroup's task queue, so now...
                            # assign a task to a worker from the workgroup's task queue
                            for worker in workgroup:
                                one_task = q.get(timeout=self.mgr_qtimeout)
                                worker.tasks.put(one_task)
                gevent.sleep(0)
        except Empty:
            self.mgr_no_work = True
            if self.mgr_should_stop:
                logger.info(f"Assigned all {name} work. I've been told I should stop.")
                self.should_stop = True
            else:
                logger.info(f"Assigned all {name} work. Awaiting more tasks to assign.") 

def _SendLoop(self):
    """Dispatch messages from the send queue to the remote server.

    Note: Messages in the queue have already been serialized into wire format.
    """
    while self.isActive:
      try:
        payload, dct = self._send_queue.get()
        queue_len = self._send_queue.qsize()
        self._varz.send_queue_size(queue_len)
        # HandleTimeout sets up the transport level timeout handling
        # for this message.  If the message times out in transit, this
        # transport will handle sending a Tdiscarded to the server.
        if self._HandleTimeout(dct): continue

        with self._varz.send_time.Measure():
          with self._varz.send_latency.Measure():
            self._socket.write(payload)
        self._varz.messages_sent()
      except Exception as e:
        self._Shutdown(e)
        break 

def test_tree_broadcast(zk, long_poll, test_application_name):
    zk.create('/huskar/config/%s/alpha/foo' % test_application_name, 'FOO',
              makepath=True)

    queue_a = long_poll()
    queue_b = long_poll()

    for queue in queue_a, queue_b:
        event = queue.get(timeout=5)
        assert event['message'] == 'all'
        assert event['body']['config'] == {
            test_application_name: {'alpha': {'foo': {'value': 'FOO'}}},
        }
        assert queue.empty()

    zk.create('/huskar/config/%s/alpha/bar' % test_application_name, 'BAR',
              makepath=True)

    for queue in queue_a, queue_b:
        event = queue.get(timeout=5)
        assert event['message'] == 'update'
        assert event['body']['config'] == {
            test_application_name: {'alpha': {'bar': {'value': 'BAR'}}},
        }
        assert queue.empty() 

def test_jira_845_regression_1(zk, test_application_name, long_poll):
    """Fix http://jira.ele.to:8088/browse/FXBUG-845."""
    zk.create(
        '/huskar/service/%s/foo/169.254.0.1_5000' % test_application_name,
        '{"ip":"169.254.0.1","port":{"main":5000}}', makepath=True)

    queue = long_poll()

    event = queue.get(timeout=5)
    assert event['message'] == 'all'

    zk.create('/huskar/service/%s/beta' % test_application_name,
              '{"link":["foo"]}', makepath=True)
    zk.create(
        '/huskar/service/%s/foo/169.254.0.2_5000' % test_application_name,
        '', makepath=True)

    event = queue.get(timeout=5)
    assert event['message'] != 'all', 'We should be silent for cluster linking'
    assert queue.empty(), 'We should notify for node creation only' 

def _HandleTimeout(self, msg_properties):
    """Determine if a message has timed out yet (because it waited in the queue
    for too long).  If it hasn't, initialize the timeout handler to fire if the
    message times out in transit.

    Args:
      msg_properties - The properties of the message.
    """
    timeout_event = msg_properties.get(Deadline.EVENT_KEY, None)
    if timeout_event and timeout_event.Get():
      # The message has timed out before it got out of the send queue
      # In this case, we can discard it immediately without even sending it.
      tag = msg_properties.pop(Tag.KEY, 0)
      if tag != 0:
        self._ReleaseTag(tag)
      return True
    elif timeout_event:
      # The event exists but hasn't been signaled yet, hook up a
      # callback so we can be notified on a timeout.
      def timeout_proc():
        timeout_tag = msg_properties.pop(Tag.KEY, 0)
        if timeout_tag:
          self._OnTimeout(timeout_tag)

      timeout_event.Subscribe(lambda evt: timeout_proc(), True)
      return False
    else:
      # No event was created, so this will never timeout.
      return False 

def listen(request):
    """
    Handles long polling connections
    ---
    get:
      tags:
      - "Client API"
      summary: "Handles long polling connections"
      description: ""
      operationId: "listen"
      produces:
      - "application/json"
      responses:
        200:
          description: "Success"
    """
    server_state = get_state()
    config = request.registry.settings
    conn_id = utils.uuid_from_string(request.params.get("conn_id"))
    connection = server_state.connections.get(conn_id)
    if not connection:
        raise HTTPUnauthorized()
    # attach a queue to connection
    connection.queue = Queue()
    connection.deliver_catchup_messages()
    request.response.app_iter = yield_response(request, connection, config)
    return request.response 

def test_logged_with_rate_limit(
        client, client_ip, mocker, test_user, test_token,
        configs, use_username):
    if '127.0.0.1' in configs:
        configs[client_ip] = configs.pop('127.0.0.1')
    if use_username:
        configs.update({
            test_user.username: {
                'rate': 1,
                'capacity': 3,
            }
        })
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', configs)

    def worker(queue):
        response = client.get('/api/need_login', headers={
            'Authorization': test_token,
        })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(10):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    assert queue.get_nowait() == 429 

def await_data(connection, config):
    messages = []
    # block for first message - wake up after a while
    try:
        messages.extend(connection.queue.get(timeout=config["wake_connections_after"]))
    except Empty:
        pass
    # get more messages if enqueued takes up total 0.25
    while True:
        try:
            messages.extend(connection.queue.get(timeout=0.25))
        except Empty:
            break
    return messages 

def queue(self):
        return gevent.queue.Queue() 

def QueueEmpty(self):
        return gevent.queue.Empty 

def _default_parser(context, response, queue):
    """默认的Response对象解析器
    """
    content_type = response.headers["content-type"]
    if content_type.startswith("application/json"):
        return response.json()
    else:
        return response.text 

def _sync_execute(self, context, start_req, parser):
        queue = list(start_req)
        result = []
        while queue:
            req = queue.pop(0)
            req = self._check_req(req)
            if req.parser is None:
                req.parser = parser
            result.append(req(context, queue))
        return result 

def test_default_anonymous_no_concurrent_limit(
        client, client_ip, switch_on, mocker, url):
    def fake_switch(name, default=True):
        if name == SWITCH_ENABLE_CONCURRENT_LIMITER:
            return switch_on
        return default

    mocker.patch.object(switch, 'is_switched_on', fake_switch)
    if not switch_on:
        mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
            '__default__': {
                'ttl': 100,
                'capacity': 1,
            }
        })

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

def __init__(self):
        self.message_queue = gevent.queue.Queue()