Python janus.Queue() Examples

def test_put_with_waiting_getters(self):
        loop = janus.current_loop()
        fut = loop.create_future()

        async def go():
            fut.set_result(None)
            ret = await q.get()
            return ret

        async def put():
            await q.put('a')

        _q = janus.Queue()
        q = _q.async_q
        t = loop.create_task(go())
        await fut
        await put()
        self.assertEqual(await t, 'a')

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def execute_write_fn(self, fn, block=False):
        task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
        if self._write_queue is None:
            self._write_queue = queue.Queue()
        if self._write_thread is None:
            self._write_thread = threading.Thread(
                target=self._execute_writes, daemon=True
            )
            self._write_thread.start()
        reply_queue = janus.Queue()
        self._write_queue.put(WriteTask(fn, task_id, reply_queue))
        if block:
            result = await reply_queue.async_q.get()
            if isinstance(result, Exception):
                raise result
            else:
                return result
        else:
            return task_id 

def test_put_cancelled(self):
        loop = janus.current_loop()
        _q = janus.Queue()
        q = _q.async_q

        async def queue_put():
            await q.put(1)
            return True

        async def test():
            return (await q.get())

        t = loop.create_task(queue_put())
        self.assertEqual(1, await test())
        self.assertTrue(t.done())
        self.assertTrue(t.result())

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def client(tmpdir, loop):
    store = PanStore(tmpdir)
    queue = janus.Queue()
    conf = ServerConfig("example", "https://exapmle.org")
    conf.history_fetch_delay = 0.1

    store.save_server_user("example", "@example:example.org")

    pan_client = PanClient(
        "example",
        store,
        conf,
        "https://example.org",
        queue.async_q,
        "@example:example.org",
        "DEVICEID",
        tmpdir,
        store_class=SqliteStore,
    )

    yield pan_client

    await pan_client.close() 

def test_float_maxsize(self):
        _q = janus.Queue(maxsize=1.3)
        q = _q.async_q
        q.put_nowait(1)
        q.put_nowait(2)
        self.assertTrue(q.full())
        self.assertRaises(asyncio.QueueFull, q.put_nowait, 3)

        _q.close()
        await _q.wait_closed()

        _q = janus.Queue(maxsize=1.3)
        q = _q.async_q

        async def queue_put():
            await q.put(1)
            await q.put(2)
            self.assertTrue(q.full())

        await queue_put()

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_get_with_putters(self):
        loop = janus.current_loop()
        _q = janus.Queue(1)
        q = _q.async_q
        q.put_nowait(1)

        fut = loop.create_future()

        async def put():
            t = asyncio.ensure_future(q.put(2))
            await asyncio.sleep(0.01)
            fut.set_result(None)
            return t

        t = await put()

        res = await q.get()
        self.assertEqual(1, res)

        await t
        self.assertEqual(1, q.qsize())

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_get_with_waiting_putters(self):
        loop = janus.current_loop()
        _q = janus.Queue(maxsize=1)
        q = _q.async_q

        loop.create_task(q.put('a'))
        loop.create_task(q.put('b'))

        await asyncio.sleep(0.01)

        self.assertEqual(await q.get(), 'a')
        self.assertEqual(await q.get(), 'b')

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_async_join_async_done(self):
        loop = janus.current_loop()
        q = janus.Queue()

        def threaded():
            for i in range(5):
                val = q.sync_q.get()
                self.assertEqual(val, i)
                q.sync_q.task_done()

        async def go():
            f = loop.run_in_executor(None, threaded)
            for i in range(5):
                await q.async_q.put(i)

            await q.async_q.join()

            await f
            self.assertTrue(q.async_q.empty())

        for i in range(3):
            await go()

        q.close()
        await q.wait_closed() 

def test_get_cancelled(self):
        loop = janus.current_loop()
        _q = janus.Queue()
        q = _q.async_q

        async def queue_get():
            return await asyncio.wait_for(q.get(), 0.051)

        async def test():
            get_task = loop.create_task(queue_get())
            await asyncio.sleep(0.01)  # let the task start
            q.put_nowait(1)
            return await get_task

        self.assertEqual(1, await test())

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_sync_join_async_done(self):
        loop = janus.current_loop()
        q = janus.Queue()

        def threaded():
            for i in range(5):
                q.sync_q.put(i)
            q.sync_q.join()

        async def go():
            f = loop.run_in_executor(None, threaded)
            for i in range(5):
                val = await q.async_q.get()
                self.assertEqual(val, i)
                q.async_q.task_done()

            self.assertTrue(q.async_q.empty())

            await f

        for i in range(3):
            await go()

        q.close()
        await q.wait_closed() 

def test_sync_put_async_join(self):
        loop = janus.current_loop()
        q = janus.Queue()

        for i in range(5):
            q.sync_q.put(i)

        async def do_work():
            await asyncio.sleep(1)
            while True:
                await q.async_q.get()
                q.async_q.task_done()

        task = loop.create_task(do_work())

        async def wait_for_empty_queue():
            await q.async_q.join()
            task.cancel()

        await wait_for_empty_queue()

        q.close()
        await q.wait_closed() 

def test_async_put_sync_get(self):
        loop = janus.current_loop()
        q = janus.Queue()

        def threaded():
            for i in range(5):
                val = q.sync_q.get()
                self.assertEqual(val, i)

        async def go():
            f = loop.run_in_executor(None, threaded)
            for i in range(5):
                await q.async_q.put(i)

            await f
            self.assertTrue(q.async_q.empty())

        for i in range(3):
            await go()

        q.close()
        await q.wait_closed() 

def test_get_cancelled_race(self):
        loop = janus.current_loop()
        _q = janus.Queue()
        q = _q.async_q

        f1 = loop.create_future()

        async def g1():
            f1.set_result(None)
            await q.get()

        t1 = loop.create_task(g1())
        t2 = loop.create_task(q.get())

        await f1
        await asyncio.sleep(0.01)
        t1.cancel()

        with self.assertRaises(asyncio.CancelledError):
            await t1
        self.assertTrue(t1.done())
        q.put_nowait('a')

        await t2
        self.assertEqual(t2.result(), 'a')

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_maxsize(self):
        loop = janus.current_loop()
        _q = janus.Queue(maxsize=2)
        q = _q.async_q
        self.assertEqual(2, q.maxsize)
        have_been_put = []

        fut = loop.create_future()

        async def putter():
            for i in range(3):
                await q.put(i)
                have_been_put.append(i)
                if i == q.maxsize - 1:
                    fut.set_result(None)
            return True

        async def test():
            t = loop.create_task(putter())
            await fut

            # The putter is blocked after putting two items.
            self.assertEqual([0, 1], have_been_put)
            self.assertEqual(0, q.get_nowait())

            # Let the putter resume and put last item.
            await t
            self.assertEqual([0, 1, 2], have_been_put)
            self.assertEqual(1, q.get_nowait())
            self.assertEqual(2, q.get_nowait())

        await test()

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_blocking_get(self):
        _q = janus.Queue()
        q = _q.async_q
        q.put_nowait(1)

        res = await q.get()
        self.assertEqual(1, res)

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_blocking_get_wait(self):
        loop = janus.current_loop()
        _q = janus.Queue()
        q = _q.async_q
        started = asyncio.Event()
        finished = False

        async def queue_get():
            nonlocal finished
            started.set()
            res = await q.get()
            finished = True
            return res

        async def queue_put():
            loop.call_later(0.01, q.put_nowait, 1)
            queue_get_task = loop.create_task(queue_get())
            await started.wait()
            self.assertFalse(finished)
            res = await queue_get_task
            self.assertTrue(finished)
            return res

        res = await queue_put()
        self.assertEqual(1, res)

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_nonblocking_get(self):
        _q = janus.Queue()
        q = _q.async_q
        q.put_nowait(1)
        self.assertEqual(1, q.get_nowait())

        _q.close()
        await _q.wait_closed() 

def test_closed(self):
        q = janus.Queue()
        self.assertFalse(q.closed)
        self.assertFalse(q.async_q.closed)
        self.assertFalse(q.sync_q.closed)
        q.close()
        self.assertTrue(q.closed)
        self.assertTrue(q.async_q.closed)
        self.assertTrue(q.sync_q.closed) 

def test_blocking_put(self):
        _q = janus.Queue()
        q = _q.async_q

        # No maxsize, won't block.
        await q.put(1)

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_blocking_put_wait(self):
        loop = janus.current_loop()
        _q = janus.Queue(maxsize=1)
        q = _q.async_q
        started = asyncio.Event()
        finished = False

        async def queue_put():
            nonlocal finished
            started.set()
            await q.put(1)
            await q.put(2)
            finished = True

        async def queue_get():
            loop.call_later(0.01, q.get_nowait)
            queue_put_task = loop.create_task(queue_put())
            await started.wait()
            self.assertFalse(finished)
            await queue_put_task
            self.assertTrue(finished)

        await queue_get()

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_nonblocking_put_exception(self):
        _q = janus.Queue(maxsize=1)
        q = _q.async_q
        q.put_nowait(1)
        self.assertRaises(asyncio.QueueFull, q.put_nowait, 2)

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_put_cancelled_race(self):
        loop = janus.current_loop()
        _q = janus.Queue(maxsize=1)
        q = _q.async_q

        put_a = loop.create_task(q.put('a'))
        put_b = loop.create_task(q.put('b'))
        put_c = loop.create_task(q.put('X'))

        await put_a
        self.assertFalse(put_b.done())

        put_c.cancel()

        with self.assertRaises(asyncio.CancelledError):
            await put_c

        async def go():
            a = await q.get()
            self.assertEqual(a, 'a')
            b = await q.get()
            self.assertEqual(b, 'b')
            self.assertTrue(put_b.done())

        await go()

        self.assertFalse(_q._sync_mutex.locked())
        _q.close()
        await _q.wait_closed() 

def test_closed_loop_non_failing(self):
        loop = janus.current_loop()
        _q = janus.Queue(QUEUE_SIZE)
        q = _q.sync_q
        # we are pacthing loop to follow setUp/tearDown agreement
        with patch.object(loop, 'call_soon_threadsafe') as func:
            func.side_effect = RuntimeError()
            q.put_nowait(1)
            self.assertEqual(func.call_count, 1)
        _q.close()
        await _q.wait_closed() 

def test_maxsize(self):
        q = janus.Queue(5)
        self.assertIs(5, q.maxsize) 

def __init__(self, name, event_base_class=None, loop=None):
        """Contructor of KytosEventBuffer receive the parameters below.

        Args:
            name (string): name of KytosEventBuffer.
            event_base_class (class): Class of KytosEvent.
        """
        self.name = name
        self._event_base_class = event_base_class
        self._loop = loop
        self._queue = Queue(loop=self._loop)
        self._reject_new_events = False 

def janus_queue_put_async(_janus_queue: Queue, func: Callable, *args, **kwargs):
    await _janus_queue.async_q.put((func, args, kwargs))


# sync put new task to janus queue 

def janus_queue_put_sync(_janus_queue: Queue, func: Callable, *args, **kwargs):
    _janus_queue.sync_q.put((func, args, kwargs)) 

def pan_proxy_server(tempdir, aiohttp_server):
    loop = asyncio.get_event_loop()
    app = web.Application()

    server_name = faker.hostname()

    config = ServerConfig(server_name, urlparse("https://example.org"))

    pan_queue = janus.Queue()
    ui_queue = janus.Queue()

    proxy = ProxyDaemon(
        config.name,
        config.homeserver,
        config,
        tempdir,
        send_queue=pan_queue.async_q,
        recv_queue=ui_queue.async_q,
        proxy=None,
        ssl=False,
        client_store_class=SqliteStore
    )

    app.add_routes([
        web.post("/_matrix/client/r0/login", proxy.login),
        web.get("/_matrix/client/r0/sync", proxy.sync),
        web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages),
        web.put(
            r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}",
            proxy.send_message
        ),
        web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter),
        web.post("/_matrix/client/r0/search", proxy.search),
        web.options("/_matrix/client/r0/search", proxy.search_opts),
    ])

    server = await aiohttp_server(app)

    yield server, proxy, (pan_queue, ui_queue)

    await proxy.shutdown(app) 

def __init__(self):
        """Initializer for AsyncClientInbox."""
        self._queue = janus.Queue() 

def setup_aio_support(cls, loop=None):
        """有些回调函数（比如 Qt UI 操作）只能在主线程中执行，
        我们这里通过 asyncio Queue 来实现。

        这个和 qt signal 的设计类似。
        """
        if loop is None:
            import asyncio
            loop = asyncio.get_event_loop()
        cls.aioqueue = janus.Queue()
        cls.worker_task = loop.create_task(Signal.worker())
        cls.has_aio_support = True