Python typing.AsyncIterable() Examples

def _messages(self) -> AsyncIterable[Any]:
        try:
            while True:
                try:
                    raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=self.MESSAGE_TIMEOUT)
                    raw_msg = ujson.loads(raw_msg_str)

                    yield raw_to_response(raw_msg)
                except asyncio.TimeoutError:
                    try:
                        pong_waiter = await self._client.ping()
                        await asyncio.wait_for(pong_waiter, timeout=self.PING_TIMEOUT)
                    except asyncio.TimeoutError:
                        raise
        except asyncio.TimeoutError:
            self.logger().warning("WebSocket ping timed out. Going to reconnect...")
            return
        except ConnectionClosed:
            return
        finally:
            await self.disconnect()

    # emit messages 

def _inner_messages(self, ws: websockets.WebSocketClientProtocol) -> AsyncIterable[str]:
        # Terminate the recv() loop as soon as the next message timed out, so the outer loop can reconnect.
        try:
            while True:
                try:
                    msg: str = await asyncio.wait_for(ws.recv(), timeout=self.MESSAGE_TIMEOUT)
                    yield msg
                except asyncio.TimeoutError:
                    try:
                        pong_waiter = await ws.ping()
                        await asyncio.wait_for(pong_waiter, timeout=self.PING_TIMEOUT)
                    except asyncio.TimeoutError:
                        raise
        except asyncio.TimeoutError:
            self.logger().warning("WebSocket ping timed out. Going to reconnect...")
            return
        except ConnectionClosed:
            return
        finally:
            await ws.close() 

def to_async_iterable(
        self, maxsize: int, return_index: bool
    ) -> tp.AsyncIterable[T]:

        # build stages first to verify reuse
        main_queue: IterableQueue[pypeln_utils.Element] = IterableQueue(
            maxsize=maxsize, total_sources=1,
        )

        built = {}

        workers: tp.List[Worker] = list(self.build(built, main_queue, main_queue))
        supervisor = Supervisor(workers=workers, main_queue=main_queue)

        async with supervisor:
            async for elem in main_queue:
                if return_index:
                    yield elem
                else:
                    yield elem.value 

def get_async_response(self, data: dict, batch_size: int) -> AsyncIterable:
        """Helper function for sending requests asynchronously if the API endpoint does not support batching

        Args:
            data: data to be passed to the API endpoint
            batch_size: requests count

        Yields:
            requests results parsed as json
        """
        loop = asyncio.get_event_loop()
        futures = [
            loop.run_in_executor(
                None,
                requests.post,
                self.url,
                None,
                {k: v[i] for k, v in data.items()}
            )
            for i in range(batch_size)
        ]
        for r in await asyncio.gather(*futures):
            yield r.json() 

def stream_block_gossip(self) -> AsyncIterable[SignedBeaconBlock]:
        async for block in self._gossiper.stream_block_gossip():
            yield block 

def to_async_iterable(
    stage: tp.Union[
        Stage[A], tp.Iterable[A], tp.AsyncIterable[A], pypeln_utils.Undefined
    ] = pypeln_utils.UNDEFINED,
    maxsize: int = 0,
    return_index: bool = False,
) -> tp.Union[tp.AsyncIterable[A], pypeln_utils.Partial[tp.AsyncIterable[A]]]:
    """
    Creates an iterable from a stage.

    Arguments:
        stage: A Stage, Iterable, or AsyncIterable.
        maxsize: The maximum number of objects the stage can hold simultaneously, if set to `0` (default) then the stage can grow unbounded.

    Returns:
        If the `stage` parameters is given then this function returns an iterable, else it returns a `Partial`.
    """

    if isinstance(stage, pypeln_utils.Undefined):
        return pypeln_utils.Partial(
            lambda stage: to_async_iterable(stage, maxsize=maxsize)
        )

    if isinstance(stage, Stage):
        iterable = stage.to_async_iterable(maxsize=maxsize, return_index=return_index)

    elif isinstance(stage, tp.AsyncIterable[A]):
        return stage
    else:
        iterable = from_iterable(stage, maxsize=maxsize).to_async_iterable(
            maxsize=maxsize, return_index=return_index
        )

    return iterable 

def map(
    f: MapFn,
    stage: tp.Union[Stage[A], tp.Iterable[A], tp.AsyncIterable[A]],
    workers: int = 1,
    maxsize: int = 0,
    timeout: float = 0,
    on_start: tp.Callable = None,
    on_done: tp.Callable = None,
) -> Stage[B]:
    ... 

def __call__(
        self, A, **kwargs
    ) -> tp.Union[tp.Iterable[B], tp.AsyncIterable[B], tp.Awaitable[tp.Iterable]]:
        ... 

def flat_map(
    f: FlatMapFn,
    stage: tp.Union[Stage[A], tp.Iterable[A], tp.AsyncIterable[A]],
    workers: int = 1,
    maxsize: int = 0,
    timeout: float = 0,
    on_start: tp.Callable = None,
    on_done: tp.Callable = None,
) -> Stage[B]:
    ... 

def __init__(
        self,
        iterable: AsyncIterable,
        callback: Callable,
        reject_callback: Optional[Callable] = None,
    ) -> None:
        self.iterator = iterable.__aiter__()
        self.callback = callback
        self.reject_callback = reject_callback
        self._close_event = Event() 

def test_iss_client_async_iterable():
    iss = client.ISSClient("test_url")
    assert isinstance(iss, typing.AsyncIterable)
    assert str(iss) == "ISSClient(url=test_url, query={})" 

def verify_stream_type(endpoint: AsyncioEndpoint) -> AsyncIterable[Event]:
    async for event in endpoint.stream(Event):
        yield event 

def test_async_iterable(self):
        base_it = range(10)  # type: Iterator[int]
        it = AsyncIteratorWrapper(base_it)
        self.assertIsInstance(it, typing.AsyncIterable)
        self.assertIsInstance(it, typing.AsyncIterable)
        self.assertNotIsInstance(42, typing.AsyncIterable) 

def stream_block_gossip(self) -> AsyncIterable[SignedBeaconBlock]:
        async with self._block_gossip:
            async for block_message in self._block_gossip:
                if block_message.from_id == self.pubsub.my_id:
                    # FIXME: this check should happen inside `py-libp2p`
                    continue
                # TODO: validate block further at the p2p layer?
                block_data = block_message.data
                yield _deserialize_gossip(block_data, SignedBeaconBlock) 

def to_async_iterable(
    maxsize: int = 0, return_index: bool = False,
) -> pypeln_utils.Partial[tp.AsyncIterable[A]]:
    ... 

def get_blocks_by_range(
        self, peer_id: PeerID, start_slot: Slot, count: int, step: int = 1
    ) -> AsyncIterable[SignedBeaconBlock]:
        request = BlocksByRangeRequest.create(
            start_slot=start_slot, count=count, step=step
        )
        if request.count_expected_blocks() < 1:
            return

        stream = await self.new_stream(
            peer_id, (ReqRespProtocol.beacon_blocks_by_range.id(),)
        )
        async for block in self._request_responder.get_blocks_by_range(stream, request):
            yield block 

def get_blocks_by_root(
        self, peer_id: PeerID, *roots: Sequence[Root]
    ) -> AsyncIterable[SignedBeaconBlock]:
        if not roots:
            return

        stream = await self.new_stream(
            peer_id, (ReqRespProtocol.beacon_blocks_by_root.id(),)
        )
        async for block in self._request_responder.get_blocks_by_root(stream, *roots):
            yield block 

def __init__(
        self,
        key_store: KeyStoreAPI,
        clock: AsyncIterable[Tick],
        beacon_node: BeaconNodeAPI,
    ) -> None:
        self._key_store = key_store
        self._clock = clock
        self._beacon_node = beacon_node

        self._duty_store = DutyStore()
        self._signature_db = InMemorySignatoryDB() 

def read_cache(start: str = None, end: str = None) -> AsyncIterable:
    """Generate the readings response data for the cached readings.

    Args:
        start: An RFC3339 formatted timestamp defining the starting
            bound on the cache data to return. An empty string or None
            designates no starting bound. (default: None)
        end: An RFC3339 formatted timestamp defining the ending
            bound on the cache data to return. An empty string or None
            designates no ending bound. (default: None)

    Yields:
        A dictionary representation of a device reading response.
    """
    logger.info('issuing command', command='READ CACHE', start=start, end=end)

    # FIXME: this could benefit from being async. this would require the plugin
    #   api client to provide async behaviors as well.
    for p in plugin.manager:
        if not p.active:
            logger.debug(
                'plugin not active, will not read its devices',
                plugin=p.tag, plugin_id=p.id,
            )
            continue

        logger.debug('getting cached readings for plugin', plugin=p.tag, command='READ CACHE')
        try:
            with p as client:
                for reading in client.read_cache(start=start, end=end):
                    yield reading_to_dict(reading)
        except Exception as e:
            raise errors.ServerError(
                'error while issuing gRPC request: read cache',
            ) from e 

def _socket_stream(self) -> AsyncIterable[str]:
        try:
            while True:
                async with timeout(MESSAGE_TIMEOUT):  # Timeouts if not receiving any messages for 10 seconds(ping)
                    conn: signalr_aio.Connection = (await self.websocket_connection())[0]
                    yield await conn.msg_queue.get()
        except asyncio.TimeoutError:
            self.logger().warning("Message recv() timed out. Going to reconnect...")
            return 

def _socket_user_stream(self, conn: signalr_aio.Connection) -> AsyncIterable[str]:
        try:
            while True:
                async with timeout(MESSAGE_TIMEOUT):
                    msg = await conn.msg_queue.get()
                    self._last_recv_time = time.time()
                    yield msg
        except asyncio.TimeoutError:
            self.logger().warning(f"Message recv() timed out. Reconnecting to Bittrex SignalR WebSocket... ") 

def on(self, name: str) -> AsyncIterable[Any]:
        self._set_event(name)

        async for msg in self._messages():
            if (msg["method"] == name):
                yield msg

    # authenticate connection and return result 

def _inner_messages(self,
                              ws: websockets.WebSocketClientProtocol) -> AsyncIterable[str]:
        """
        Generator function that returns messages from the web socket stream
        :param ws: current web socket connection
        :returns: message in AsyncIterable format
        """
        # Terminate the recv() loop as soon as the next message timed out, so the outer loop can reconnect.
        try:
            while True:
                try:
                    msg: str = await asyncio.wait_for(ws.recv(), timeout=Constants.MESSAGE_TIMEOUT)
                    yield msg
                except asyncio.TimeoutError:
                    try:
                        pong_waiter = await ws.ping()
                        await asyncio.wait_for(pong_waiter, timeout=Constants.PING_TIMEOUT)
                    except asyncio.TimeoutError:
                        raise
        except asyncio.TimeoutError:
            self.logger().warning("WebSocket ping timed out. Going to reconnect...")
            return
        except ConnectionClosed:
            return
        finally:
            await ws.close() 

def upload_media(self, data: Union[bytes, AsyncIterable[bytes]],
                           mime_type: Optional[str] = None, filename: Optional[str] = None,
                           size: Optional[int] = None) -> ContentURI:
        """
        Upload a file to the content repository.

        See also: `API reference <https://matrix.org/docs/spec/client_server/r0.4.0.html#post-matrix-media-r0-upload>`__

        Args:
            data: The data to upload.
            mime_type: The MIME type to send with the upload request.
            filename: The filename to send with the upload request.
            size: The file size to send with the upload request.

        Returns:
            The MXC URI to the uploaded file.

        Raises:
            MatrixResponseError: If the response does not contain a ``content_uri`` field.
        """
        if magic and isinstance(data, bytes):
            mime_type = mime_type or magic.from_buffer(data, mime=True)
        headers = {}
        if mime_type:
            headers["Content-Type"] = mime_type
        if size:
            headers["Content-Length"] = str(size)
        query = {}
        if filename:
            query["filename"] = filename
        resp = await self.api.request(Method.POST, MediaPath.upload, content=data,
                                      headers=headers, query_params=query)
        try:
            return resp["content_uri"]
        except KeyError:
            raise MatrixResponseError("`content_uri` not in response.") 

def from_async_iterable(iterable: AsyncIterable[T]) -> AsyncObservable[T]:
    """Convert an async iterable to a source stream.

    2 - xs = from_async_iterable(async_iterable)

    Returns the source stream whose elements are pulled from the
    given (async) iterable sequence."""

    return FromAsyncIterable(iterable) 

def to_async_iterable(source: AsyncObservable) -> AsyncIterable:
    """Skip the specified number of values.

    Keyword arguments:
    count -- The number of elements to skip before returning the
        remaining values.

    Returns a source stream that contains the values that occur
    after the specified index in the input source stream.
    """

    return ToAsyncIterable(source) 

def repeat(value, times=None) -> AsyncIterable:
    for value in itertools.repeat(value, times):
        yield value 

def range(*args) -> AsyncIterable:
    for value in builtins.range(*args):
        yield value 

def filter(predicate, source: AsyncIterable) -> AsyncIterable:
    async for value in source:
        if predicate(value):
            yield value 

def __init__(self, source: AsyncIterable) -> None:
        self._source = source