Python asyncio.QueueEmpty() Examples

def get(self) -> ParsedAnswer:

        try:
            item = self._queue.get_nowait()
        except asyncio.QueueEmpty:
            item = await self._queue.get()

        self._queue.task_done()

        # If we receive an exception when reading the queue, we raise it
        if isinstance(item, Exception):
            self._closed = True
            raise item

        # Don't need to save new answers or
        # send the stop message if we already received the complete message
        answer_type, execution_result = item
        if answer_type == "complete":
            self.send_stop = False
            self._closed = True

        return item 

def __handle_queue_update(cls,
                              q: Union[queue.Queue, asyncio.Queue],
                              first: object) -> list:
        ret = first if isinstance(first, list) else [first]

        while True:
            try:
                elem = q.get_nowait()
                if isinstance(elem, list):
                    ret.extend(elem)
                else:
                    ret.append(elem)
            except (asyncio.QueueEmpty, queue.Empty):
                break

        return ret 

def _stream_protocol_messages(self,
                                        protocol_class: Type[ProtocolAPI],
                                        ) -> AsyncIterator[CommandAPI[Any]]:
        """
        Stream the messages for the specified protocol.
        """
        async with self._protocol_locks[protocol_class]:
            self.raise_if_streaming_error()
            msg_queue = self._protocol_queues[protocol_class]
            while self.is_streaming:
                try:
                    # We use an optimistic strategy here of using
                    # `get_nowait()` to reduce the number of times we yield to
                    # the event loop.  Since this is an async generator it will
                    # yield to the loop each time it returns a value so we
                    # don't have to worry about this blocking other processes.
                    yield msg_queue.get_nowait()
                except asyncio.QueueEmpty:
                    yield await msg_queue.get()

    #
    # Message reading and streaming API
    # 

def raise_errors(self):
        # If the error monitor is running then just return, as that means we are
        # running as a worker and so can rely on the error monitor to pickup the
        # errors which an happen in the various background tasks
        if self._error_monitor_lock.locked():
            return

        # Check queue for errors
        try:
            error: Error = self.error_queue.get_nowait()
        except asyncio.QueueEmpty:
            # No errors, everything ok
            return
        else:
            # If there is an error then raise it
            raise error.value 

def run(self, *args):
        while True:
            msg = await self.queue.get()
            async with self.get_connection() as conn:
                self.connect_counter += 1
                while True:
                    try:
                        await self.send_message(msg, conn)
                        self.mail_success += 1
                    except:
                        self.mail_failed += 1
                        await conn.close()
                        logger.exception('Mailer reconnect')
                        self.reconnect_counter += 1
                        await asyncio.sleep(2)
                        await conn.open()
                    try:
                        msg = self.queue.get_nowait()
                    except asyncio.QueueEmpty:
                        break 

def receive_until(self, monotonic_deadline: float) -> typing.Optional[pyuavcan.transport.TransferFrom]:
        try:
            timeout = monotonic_deadline - self._loop.time()
            if timeout > 0:
                transfer = await asyncio.wait_for(self._queue.get(), timeout, loop=self._loop)
            else:
                transfer = self._queue.get_nowait()
        except (asyncio.TimeoutError, asyncio.QueueEmpty):
            # If there are unprocessed transfers, allow the caller to read them even if the instance is closed.
            if self._maybe_finalizer is None:
                raise pyuavcan.transport.ResourceClosedError(f'{self} is closed')
            return None
        else:
            assert isinstance(transfer, pyuavcan.transport.TransferFrom), 'Internal protocol violation'
            assert transfer.source_node_id == self._specifier.remote_node_id or self._specifier.remote_node_id is None
            return transfer 

def get(
        self, *, no_ack=False, fail=True, timeout=5
    ) -> Optional[IncomingMessage]:

        """ Get message from the queue.

        :param no_ack: if :class:`True` you don't need to call
                       :func:`aio_pika.message.IncomingMessage.ack`
        :param timeout: execution timeout
        :param fail: Should return :class:`None` instead of raise an
                     exception :class:`aio_pika.exceptions.QueueEmpty`.
        :return: :class:`aio_pika.message.IncomingMessage`
        """

        msg = await asyncio.wait_for(
            self.channel.basic_get(self.name, no_ack=no_ack), timeout=timeout,
        )  # type: Optional[DeliveredMessage]

        if msg is None:
            if fail:
                raise QueueEmpty
            return

        return IncomingMessage(msg, no_ack=no_ack) 

def close(self):
        if not self._consumer_tag:
            return

        await self._amqp_queue.cancel(self._consumer_tag)
        self._consumer_tag = None

        def get_msg():
            try:
                return self._queue.get_nowait()
            except asyncio.QueueEmpty:
                return

        # Reject all messages
        msg = get_msg()  # type: IncomingMessage
        while msg and not self._amqp_queue.channel.closing.done():
            await msg.reject(requeue=True)
            msg = get_msg()  # type: IncomingMessage 

def main(self):
        print('start')
        async_q = self._queue.async_q
        main_loop = asyncio.get_event_loop()
        while not (self._stopped and async_q.empty()):

            try:
                event = self.queue.get_nowait()
            except asyncio.QueueEmpty:
                pass
            else:
                asyncio.run_coroutine_threadsafe(
                    self.event_hadler(event),
                    main_loop
                )
                async_q.task_done()
            await asyncio.sleep(0.0001) 

def handle_get_request(self, environ):
        """Handle a long-polling GET request from the client."""
        connections = [
            s.strip()
            for s in environ.get('HTTP_CONNECTION', '').lower().split(',')]
        transport = environ.get('HTTP_UPGRADE', '').lower()
        if 'upgrade' in connections and transport in self.upgrade_protocols:
            self.server.logger.info('%s: Received request to upgrade to %s',
                                    self.sid, transport)
            return await getattr(self, '_upgrade_' + transport)(environ)
        if self.upgrading or self.upgraded:
            # we are upgrading to WebSocket, do not return any more packets
            # through the polling endpoint
            return [packet.Packet(packet.NOOP)]
        try:
            packets = await self.poll()
        except exceptions.QueueEmpty:
            exc = sys.exc_info()
            await self.close(wait=False)
            six.reraise(*exc)
        return packets 

def poll(self):
        """Wait for packets to send to the client."""
        try:
            packets = [await asyncio.wait_for(self.queue.get(),
                                              self.server.ping_timeout)]
            self.queue.task_done()
        except (asyncio.TimeoutError, asyncio.CancelledError):
            raise exceptions.QueueEmpty()
        if packets == [None]:
            return []
        while True:
            try:
                pkt = self.queue.get_nowait()
                self.queue.task_done()
                if pkt is None:
                    self.queue.put_nowait(None)
                    break
                packets.append(pkt)
            except asyncio.QueueEmpty:
                break
        return packets 

def get_available_candidate(self) -> discord.TextChannel:
        """
        Return a dormant channel to turn into an available channel.

        If no channel is available, wait indefinitely until one becomes available.
        """
        log.trace("Getting an available channel candidate.")

        try:
            channel = self.channel_queue.get_nowait()
        except asyncio.QueueEmpty:
            log.info("No candidate channels in the queue; creating a new channel.")
            channel = await self.create_dormant()

            if not channel:
                log.info("Couldn't create a candidate channel; waiting to get one from the queue.")
                await self.notify()
                channel = await self.wait_for_dormant_channel()

        return channel 

def ws_events_current_operation(request, ws, user):
    if not await valid_origin_header(request):
        return
    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "newoperationeventlog";')
                    if user['current_operation'] != "":
                        query = await db_model.operation_query()
                        operation = await db_objects.get(query, name=user['current_operation'])
                        query = await db_model.operationeventlog_query()
                        initial_events = await db_objects.execute(query.where(db_model.OperationEventLog.operation == operation))
                        for i in initial_events:
                            await ws.send(js.dumps(i.to_json()))
                        await ws.send("")
                        while True:
                            try:
                                msg = conn.notifies.get_nowait()
                                id = (msg.payload)

                                t = await db_objects.get(query, id=id)
                                if t.operation == operation:
                                    await ws.send(js.dumps(t.to_json()))
                            except asyncio.QueueEmpty as e:
                                await asyncio.sleep(0.5)
                                await ws.send("")  # this is our test to see if the client is still there
                            except Exception as e:
                                print(e)
                                continue
    finally:
        pool.close() 

def get(self) -> T:
        """Remove and return an item from the queue.

        If queue is empty, wait until an item is available.
        """
        with self._mutex:
            try:
                return self.get_nowait()
            except asyncio.QueueEmpty:
                pass

            getter = asyncio.get_event_loop().create_future()
            self._getters.append(getter)

        while True:
            try:
                await getter
            except:
                with self._mutex:
                    getter.cancel()  # Just in case getter is not done yet.
                    try:
                        # Clean self._getters from canceled getters.
                        self._getters.remove(getter)
                    except ValueError:
                        # The getter could be removed from self._getters by a
                        # previous put_nowait call.
                        pass
                    if not self.empty() and not getter.cancelled():
                        # We were woken up by put_nowait(), but can't take
                        # the call.  Wake up the next in line.
                        self._wakeup_next(self._getters)
                raise

            with self._mutex:
                try:
                    return self.get_nowait()
                except asyncio.QueueEmpty:
                    getter = asyncio.get_event_loop().create_future()
                    self._getters.append(getter) 

def ws_events_new_current_operation(request, ws, user):
    if not await valid_origin_header(request):
        return
    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "newoperationeventlog";')
                    if user['current_operation'] != "":
                        query = await db_model.operation_query()
                        operation = await db_objects.get(query, name=user['current_operation'])
                        query = await db_model.operationeventlog_query()
                        while True:
                            try:
                                msg = conn.notifies.get_nowait()
                                id = (msg.payload)

                                t = await db_objects.get(query, id=id)
                                if t.operation == operation:
                                    await ws.send(js.dumps(t.to_json()))
                            except asyncio.QueueEmpty as e:
                                await asyncio.sleep(0.5)
                                await ws.send("")  # this is our test to see if the client is still there
                            except Exception as e:
                                print(e)
                                continue
    finally:
        pool.close()


# -------------- EVENT LOGS ---------------------- 

def ws_graph_edges_current_operation(request, ws, user):
    if not await valid_origin_header(request):
        return
    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "newcallbackgraphedge";')
                    await cur.execute('LISTEN "updatedcallbackgraphedge";')
                    if user['current_operation'] != "":
                        query = await db_model.operation_query()
                        operation = await db_objects.get(query, name=user['current_operation'])
                        query = await db_model.callbackgraphedge_query()
                        initial_edges = await db_objects.execute(query.where(
                            (db_model.CallbackGraphEdge.operation == operation) &
                            (db_model.CallbackGraphEdge.end_timestamp == None)
                        ))
                        for i in initial_edges:
                            await ws.send(js.dumps(i.to_json()))
                        await ws.send("")
                        while True:
                            try:
                                msg = conn.notifies.get_nowait()
                                id = (msg.payload)

                                t = await db_objects.get(query, id=id, operation=operation)
                                if t.operation == operation:
                                    await ws.send(js.dumps(t.to_json()))
                            except asyncio.QueueEmpty as e:
                                await asyncio.sleep(0.5)
                                await ws.send("")  # this is our test to see if the client is still there
                            except Exception as e:
                                print(e)
                                continue
    finally:
        pool.close()


# CHECK ORIGIN HEADERS FOR WEBSOCKETS 

def _get_connection(self):
        # if there are no connections, let QueueEmpty exception happen
        # if all connections are closed, remove all of them and let QueueEmpty exception happen
        while True:
            connection = self._idle_connections.get_nowait()  # type: ClientConnection

            if connection.open:
                logger.debug(f'Reusing connection {id(connection)} to: {self.host}:{self.port}')
                return connection 

def get_nowait(self) -> T:
        """Remove and return an item from the queue.

        Return an item if one is immediately available, else raise QueueEmpty.
        """
        with self._mutex:
            if self.empty():
                raise asyncio.QueueEmpty
            item = self._get()
            self._wakeup_next(self._putters)
            return item 

def create_queue(self):
        """Create a queue object."""
        q = asyncio.Queue()
        q.Empty = asyncio.QueueEmpty
        return q 

def decrement_payload_size(self):
        "Decrement chunk size and flush chunks currently in queue"
        self.payload_size -= 1
        # Empty the queue
        while True:
            try:
                self.sample_queue.get_nowait()
            except asyncio.QueueEmpty:
                break
        return self.audio_config.chunk_size + self.HEADER_SIZE 

def ws_keylogs_current_operation(request, ws, user):
    if not await valid_origin_header(request):
        return

    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "newkeylog";')
                    # BEFORE WE START GETTING NEW THINGS, UPDATE WITH ALL OF THE OLD DATA
                    query = await db_model.operation_query()
                    operation = await db_objects.get(query, name=user['current_operation'])
                    # now pull off any new payloads we got queued up while processing old data
                    while True:
                        try:
                            msg = conn.notifies.get_nowait()
                            id = (msg.payload)
                            try:
                                query = await db_model.keylog_query()
                                c = await db_objects.get(query, id=id, operation=operation)
                                await ws.send(js.dumps({**c.to_json()}))
                            except Exception as e:
                                pass  # we got a file that's just not part of our current operation, so move on
                        except asyncio.QueueEmpty as e:
                            await asyncio.sleep(2)
                            await ws.send("")  # this is our test to see if the client is still there
                            continue
                        except Exception as e:
                            print(e)
                            continue
    finally:
        pool.close()


# ------ OPERATING COMMAND POPUP INFORMATION --------------------
# ----- INCLUDES CREDENTIALS, PAYLOADS, PAYLOADSONHOST ------------
# notifications for new credentials 

def ws_credentials_current_operation(request, ws, user):
    if not await valid_origin_header(request):
        return

    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "newcredential";')
                    # BEFORE WE START GETTING NEW THINGS, UPDATE WITH ALL OF THE OLD DATA
                    query = await db_model.operation_query()
                    operation = await db_objects.get(query, name=user['current_operation'])
                    query = await db_model.credential_query()
                    creds = await db_objects.execute(query.where(Credential.operation == operation))
                    for c in creds:
                        await ws.send(js.dumps({**c.to_json()}))
                    await ws.send("")
                    # now pull off any new payloads we got queued up while processing old data
                    while True:
                        try:
                            msg = conn.notifies.get_nowait()
                            id = (msg.payload)
                            try:
                                query = await db_model.credential_query()
                                c = await db_objects.get(query, id=id, operation=operation)
                                await ws.send(js.dumps({**c.to_json()}))
                            except Exception as e:
                                pass  # we got a file that's just not part of our current operation, so move on
                        except asyncio.QueueEmpty as e:
                            await asyncio.sleep(2)
                            await ws.send("")  # this is our test to see if the client is still there
                            continue
                        except Exception as e:
                            print(e)
                            continue
    finally:
        pool.close()

# ------------- KEYLOG ---------------------------
# notifications for new keylogs 

def ws_manual_files_current_operation(request, ws, user):
    if not await valid_origin_header(request):
        return

    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "newfilemeta";')
                    await cur.execute('LISTEN "updatedfilemeta";')
                    # BEFORE WE START GETTING NEW THINGS, UPDATE WITH ALL OF THE OLD DATA
                    query = await db_model.operation_query()
                    operation = await db_objects.get(query, name=user['current_operation'])
                    query = await db_model.filemeta_query()
                    files = await db_objects.prefetch(query.where(
                        (FileMeta.operation == operation) & (FileMeta.deleted == False)).order_by(FileMeta.id), Task.select(), Command.select(), Callback.select())
                    for f in files:
                        if f.task is None:
                            await ws.send(js.dumps({**f.to_json()}))
                    await ws.send("")
                    # now pull off any new payloads we got queued up while processing old data
                    while True:
                        try:
                            msg = conn.notifies.get_nowait()
                            id = (msg.payload)
                            query = await db_model.filemeta_query()
                            f = await db_objects.get(query, id=id, operation=operation, deleted=False)
                            await ws.send(js.dumps({**f.to_json()}))
                        except asyncio.QueueEmpty as e:
                            await asyncio.sleep(1)
                            await ws.send("")  # this is our test to see if the client is still there
                            continue
                        except Exception as e:
                            print(e)
                            continue
    finally:
        pool.close()


# ------------- CREDENTIAL ---------------------------
# notifications for new credentials 

def ws_updated_screenshots(request, ws, user):
    if not await valid_origin_header(request):
        return

    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "updatedfilemeta";')
                    # BEFORE WE START GETTING NEW THINGS, UPDATE WITH ALL OF THE OLD DATA
                    while True:
                        try:
                            msg = conn.notifies.get_nowait()
                            id = (msg.payload)
                            query = await db_model.filemeta_query()
                            f = await db_objects.get(query, id=id)
                            if "{}/downloads/".format(user['current_operation']) in f.path and "/screenshots" in f.path:
                                if f.task:
                                    query = await db_model.task_query()
                                    task = await db_objects.get(query, id=f.task)
                                    callback_id = task.callback.id
                                    await ws.send(js.dumps({**task.callback.to_json(), **f.to_json(), 'callback_id': callback_id, 'comment': task.comment}))
                                else:
                                    await ws.send(js.dumps({**f.to_json(), 'callback_id': 0, 'comment': ''}))
                        except asyncio.QueueEmpty as e:
                            await asyncio.sleep(2)
                            await ws.send("")  # this is our test to see if the client is still there
                            continue
                        except Exception as e:
                            print(e)
                            continue
    finally:
        pool.close()


# notifications for new files in the current operation 

def ws_transform_code(request, ws, user):
    if not await valid_origin_header(request):
        return
    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "newtransformcode";')
                    await cur.execute('LISTEN "updatedtransformcode";')
                    query = await db_model.transformcode_query()
                    tc = await db_objects.execute(query)
                    for transform in tc:
                        await ws.send(js.dumps({**transform.to_json(), 'channel': "newtransformcode"}))
                    await ws.send("")
                    while True:
                        try:
                            msg = conn.notifies.get_nowait()
                            id = (msg.payload)
                            transform = await db_objects.get(query, id=id)
                            await ws.send(js.dumps({**transform.to_json(), 'channel': msg.channel}))
                        except asyncio.QueueEmpty as e:
                            await asyncio.sleep(2)
                            await ws.send("")  # this is our test to see if the client is still there
                            continue
                        except Exception as e:
                            print(e)
                            continue
    finally:
        pool.close()


# ------------- FILEMETA ---------------------------
# notifications for new screenshots 

def ws_commands(request, ws):
    if not await valid_origin_header(request):
        return

    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "newcommand";')
                    # BEFORE WE START GETTING NEW THINGS, UPDATE WITH ALL OF THE OLD DATA
                    query = await db_model.command_query()
                    commands = await db_objects.execute(query.where(Command.deleted == False))
                    for c in commands:
                        await ws.send(js.dumps(c.to_json()))
                    await ws.send("")
                    # now pull off any new payloads we got queued up while processing old data
                    while True:
                        try:
                            msg = conn.notifies.get_nowait()
                            id = (msg.payload)
                            p = await db_objects.get(query, id=id, deleted=False)
                            await ws.send(js.dumps(p.to_json()))
                        except asyncio.QueueEmpty as e:
                            await asyncio.sleep(2)
                            await ws.send("")  # this is our test to see if the client is still there
                            continue
                        except Exception as e:
                            print(e)
                            continue
    finally:
        pool.close()


# notifications for new commands 

def ws_payloadtypes(request, ws):
    if not await valid_origin_header(request):
        return

    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "newpayloadtype";')
                    await cur.execute('LISTEN "updatedpayloadtype";')
                    # BEFORE WE START GETTING NEW THINGS, UPDATE WITH ALL OF THE OLD DATA
                    query = await db_model.payloadtype_query()
                    payloadtypes = await db_objects.execute(query.where(db_model.PayloadType.deleted == False).order_by(PayloadType.id))
                    for p in payloadtypes:
                        await ws.send(js.dumps(p.to_json()))
                    await ws.send("")
                    # now pull off any new payloads we got queued up while processing old data
                    while True:
                        try:
                            msg = conn.notifies.get_nowait()
                            id = (msg.payload)
                            query = await db_model.payloadtype_query()
                            p = await db_objects.get(query, id=id, deleted=False)
                            await ws.send(js.dumps(p.to_json()))
                        except asyncio.QueueEmpty as e:
                            await asyncio.sleep(2)
                            await ws.send("")  # this is our test to see if the client is still there
                            continue
                        except Exception as e:
                            print(e)
                            continue
    finally:
        pool.close()


# ---------------- COMMANDS --------------------------
# notifications for new commands 

def ws_updated_operators(request, ws):
    if not await valid_origin_header(request):
        return

    try:
        async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor() as cur:
                    await cur.execute('LISTEN "updatedoperator";')
                    # just want updates, not anything else
                    while True:
                        # msg = await conn.notifies.get()
                        try:
                            msg = conn.notifies.get_nowait()
                            # print("got an update for a callback")
                            id = (msg.payload)
                            query = await db_model.operator_query()
                            cb = await db_objects.get(query, id=id, deleted=False)
                            await ws.send(js.dumps(cb.to_json()))
                        except asyncio.QueueEmpty as e:
                            await asyncio.sleep(2)
                            await ws.send("") # this is our test to see if the client is still there
                            continue
                        except Exception as e:
                            print(e)
                            continue
    finally:
        pool.close()


# ---------------- PAYLOADTYPES --------------------------
# notifications for new payloadtypes 

def test_nonblocking_get_exception(self):
        q = asyncio.Queue(loop=self.loop)
        self.assertRaises(asyncio.QueueEmpty, q.get_nowait) 

def test_empty_poll(self):
        mock_server = self._get_mock_server()
        s = asyncio_socket.AsyncSocket(mock_server, 'sid')
        with pytest.raises(exceptions.QueueEmpty):
            _run(s.poll())