Python typing.AsyncIterator() Examples

def predict(
        self, records: AsyncIterator[Record]
    ) -> AsyncIterator[Record]:
        if not self.model:
            raise ModelNotTrained(
                "Train the model first before getting preictions"
            )
        test_records = await self.get_test_records(records)
        x_test = pd.DataFrame([record.features() for record in test_records])
        predictions = await self.get_predictions(x_test)
        probability = await self.get_probabilities(x_test)
        target = self.parent.config.predict.name
        for record, predict, prob in zip(
            test_records, predictions, probability
        ):
            record.predicted(target, predict, max(prob))
            yield record 

def timeRangeAsync(
        start: Union[time_, datetime],
        end: Union[time_, datetime],
        step: float) -> AsyncIterator[datetime]:
    """Async version of :meth:`timeRange`."""
    assert step > 0
    delta = timedelta(seconds=step)
    t = _fillDate(start)
    tz = timezone.utc if t.tzinfo else None
    now = datetime.now(tz)
    while t < now:
        t += delta
    while t <= _fillDate(end):
        await waitUntilAsync(t)
        yield t
        t += delta 

def iter(self, offs: int) -> AsyncIterator[Any]:
        '''
        Returns an iterator of change entries in the log
        '''
        if not self.donexslog:
            return

        if self.isfini:
            raise s_exc.IsFini()

        maxoffs = offs

        for item in self._nexuslog.iter(offs):
            if self.isfini:
                raise s_exc.IsFini()
            maxoffs = item[0] + 1
            yield item

        async with self.getChangeDist(maxoffs) as dist:
            async for item in dist:
                if self.isfini:
                    raise s_exc.IsFini()
                yield item 

def getCortexAndProxy(self) -> AsyncIterator[Tuple[Any, Any]]:
        with syntest.getTestDir(startdir=self.tmpdir) as dirn:

            s_tools_backup.backup(self.testdata.dirn, dirn, compact=False)

            async with await s_cortex.Cortex.anit(dirn, conf=self.coreconfig) as core:
                assert not core.inaugural

                lockmemory = self.coreconfig.get('layers:lockmemory', False)
                logedits = self.coreconfig.get('layers:logedits', True)

                await core.view.layers[0].layrinfo.set('lockmemory', lockmemory)
                await core.view.layers[0].layrinfo.set('logedits', logedits)

            async with await s_cortex.Cortex.anit(dirn, conf=self.coreconfig) as core:
                async with core.getLocalProxy() as prox:
                    if lockmemory:
                        await core.view.layers[0].layrslab.lockdoneevent.wait()

                    await s_lmdbslab.Slab.syncLoopOnce()

                    yield core, prox 

def from_annotations(request_annotation, response_annotation):
        if (hasattr(request_annotation, "__origin__") and
                issubclass(request_annotation.__origin__, collections.abc.AsyncIterator)):
            request_type = request_annotation.__args__[0]
            request_stream = True
        else:
            request_type = request_annotation
            request_stream = False

        if (hasattr(response_annotation, "__origin__") and
                issubclass(response_annotation.__origin__, collections.abc.AsyncIterator)):
            response_type = response_annotation.__args__[0]
            response_stream = True
        else:
            response_type = response_annotation
            response_stream = False
        cardinality = Cardinality.get_cardinality_for(request_stream=request_stream,
                                                      response_stream=response_stream)
        return RPCSignature(cardinality, request_type, response_type) 

def mock_request_response(request_type: Type[BaseEvent],
                                response: BaseEvent,
                                event_bus: EndpointAPI) -> AsyncIterator[None]:
    ready = asyncio.Event()
    task = asyncio.ensure_future(_do_mock_response(request_type, response, event_bus, ready))
    await ready.wait()
    try:
        yield
    finally:
        if not task.done():
            task.cancel()

        try:
            await task
        except asyncio.CancelledError:
            pass 

def predict(
        self, records: AsyncIterator[Record]
    ) -> AsyncIterator[Tuple[Record, Any, float]]:
        if not self._filepath.is_file():
            raise ModelNotTrained("Train model before prediction.")
        async for record in records:
            record_data = []
            for feature in record.features(self.features).values():
                record_data.extend(
                    [feature] if self.np.isscalar(feature) else feature
                )
            predict = self.np.array([record_data])
            self.logger.debug(
                "Predicted Value of {} for {}: {}".format(
                    self.parent.config.predict,
                    predict,
                    self.clf.predict(predict),
                )
            )
            target = self.parent.config.predict.name
            record.predicted(
                target, self.clf.predict(predict)[0], self.confidence
            )
            yield record 

def _incoming_from_rtm(
        self, url: str, bot_id: str
    ) -> AsyncIterator[events.Event]:
        """
        Connect and discard incoming RTM event if necessary.

        :param url: Websocket url
        :param bot_id: Bot ID
        :return: Incoming events
        """
        async for data in self._rtm(url):
            event = events.Event.from_rtm(json.loads(data))
            if sansio.need_reconnect(event):
                break
            elif sansio.discard_event(event, bot_id):
                continue
            else:
                yield event 

def rtm(
        self, url: Optional[str] = None, bot_id: Optional[str] = None
    ) -> AsyncIterator[events.Event]:
        """
        Iterate over event from the RTM API

        Args:
            url: Websocket connection url
            bot_id: Connecting bot ID

        Returns:
            :class:`slack.events.Event` or :class:`slack.events.Message`

        """
        while True:
            bot_id = bot_id or await self._find_bot_id()
            url = url or await self._find_rtm_url()
            async for event in self._incoming_from_rtm(url, bot_id):
                yield event
            url = None 

def predict_input_fn(self, records: AsyncIterator[Record], **kwargs):
        """
        Uses the numpy input function with data from record features.
        """
        x_cols: Dict[str, Any] = {feature: [] for feature in self.features}
        ret_records = []
        async for record in records:
            if not record.features(self.features):
                continue
            ret_records.append(record)
            for feature, results in record.features(self.features).items():
                x_cols[feature].append(self.np.array(results))
        for feature in x_cols:
            x_cols[feature] = self.np.array(x_cols[feature])
        self.logger.info("------ Record Data ------")
        self.logger.info("x_cols:    %d", len(list(x_cols.values())[0]))
        self.logger.info("-----------------------")
        input_fn = self.tf.compat.v1.estimator.inputs.numpy_input_fn(
            x_cols, shuffle=False, num_epochs=1, **kwargs
        )
        return input_fn, ret_records 

def predict(
        self, records: AsyncIterator[Record]
    ) -> AsyncIterator[Tuple[Record, Any, float]]:
        # Load saved regression line
        regression_line = self.storage.get("regression_line", None)
        # Ensure the model has been trained before we try to make a prediction
        if regression_line is None:
            raise ModelNotTrained("Train model before prediction.")
        # Expand the regression_line into named variables
        m, b, accuracy = regression_line
        # Iterate through each record that needs a prediction
        async for record in records:
            # Grab the x data from the record
            x = record.feature(self.features[0])
            # Calculate y
            y = m * x + b
            # Set the calculated value with the estimated accuracy
            record.predicted(self.config.predict.name, y, accuracy)
            # Yield the record to the caller
            yield record 

def __aiter__(self) -> tp.AsyncIterator[T]:

        while not self.is_done():

            if self.namespace.exception:
                exception, trace = await self.exception_queue.get()

                try:
                    exception = exception(f"\n\n{trace}")
                except:
                    exception = Exception(f"\n\nOriginal: {exception}\n\n{trace}")

                raise exception

            x = await self.get()

            if isinstance(x, pypeln_utils.Continue):
                continue
            elif isinstance(x, pypeln_utils.Done):
                self.namespace.remaining -= 1
                continue

            yield x 

def on_dog_added(
    parent: typing.Any, args: dict, ctx: dict, info: dict
) -> typing.AsyncIterator[dict]:
    pubsub: PubSub = ctx["pubsub"]
    queue: Queue = Queue()

    @pubsub.on("dog_added")
    def on_dog(dog: Dog) -> None:
        queue.put(dog)

    while True:
        try:
            dog = queue.get_nowait()
        except Empty:
            await asyncio.sleep(0.01)
            continue
        else:
            queue.task_done()
            if dog is None:
                break
            yield {"dogAdded": dog._asdict()} 

def records(self) -> AsyncIterator[Record]:
        async for record in self.sctx.records():
            async for ctx, result in self.octx.run(
                [
                    Input(
                        value=record.feature(feature.name),
                        definition=Definition(
                            name=feature.name, primitive=str(feature.dtype())
                        ),
                    )
                    for feature in self.parent.config.features
                ]
            ):
                if result:
                    record.evaluated(result)
                yield record 

def init_refresh_token(
        base_url: str,
        client_id: str,
        client_secret: str,
        verify_ssl: bool,
        proxy: bool
) -> AsyncIterator[RefreshToken]:
    """Initializes RefreshToken instance and authenticates with CrowdStrike Falcon.

    Args:
        base_url (str): CrowdStrike Falcon Cloud base URL.
        client_id (str): CrowdStrike Falcon application ID.
        client_secret (str): CrowdStrike Falcon application secret.
        verify_ssl (bool): Whether the request should verify the SSL certificate.
        proxy (bool): Whether to run the integration using the system proxy.

    Yields:
        AsyncIterator[RefreshToken]: RefreshToken instance initialized with client details.
    """
    refresh_token = RefreshToken(base_url, client_id, client_secret, verify_ssl, proxy)
    await refresh_token.get_access_token()
    task = create_task(refresh_token.refresh_token_loop())
    yield refresh_token
    task.cancel() 

def lock(self, resource: TResource) -> AsyncIterator[None]:
        if resource not in self._locks:
            self._locks[resource] = asyncio.Lock()

        try:
            self._reference_counts[resource] += 1
            async with self._locks[resource]:
                yield
        finally:
            self._reference_counts[resource] -= 1
            if self._reference_counts[resource] <= 0:
                del self._reference_counts[resource]
                del self._locks[resource] 

def task_group(self) -> AsyncIterator[TaskGroup]:
        async with create_task_group() as group:
            await group.spawn(self._render_loop)
            yield group 

def apply(self, connection: ConnectionAPI) -> AsyncIterator[asyncio.Future[None]]:
        service = PingAndDisconnectIfIdle(connection, self.idle_timeout)
        async with background_asyncio_service(service) as manager:
            yield asyncio.create_task(manager.wait_finished()) 

def __anext__(self):
        limit = self._params.get('limit', None)

        count = 0
        async for page in cast(AsyncIterator[Any], self.pages()):
            for item in page:
                yield item

                count += 1
                if limit is not None and count >= limit:
                    return 

def infinite_watch(
        *,
        settings: configuration.OperatorSettings,
        resource: resources.Resource,
        namespace: Optional[str],
        freeze_mode: Optional[primitives.Toggle] = None,
) -> AsyncIterator[bodies.RawEvent]:
    """
    Stream the watch-events infinitely.

    This routine is extracted because it is difficult to test infinite loops.
    It is made as simple as possible, and is assumed to work without testing.

    This routine never ends gracefully. If a watcher's stream fails,
    a new one is recreated, and the stream continues.
    It only exits with unrecoverable exceptions.
    """
    while True:
        stream = streaming_watch(
            settings=settings,
            resource=resource,
            namespace=namespace,
            freeze_mode=freeze_mode,
        )
        async for raw_event in stream:
            yield raw_event
        await asyncio.sleep(settings.watching.reconnect_backoff) 

def get_client(app: typing.Callable) -> typing.AsyncIterator:
    async with LifespanManager(app):
        async with httpx.AsyncClient(app=app, base_url="http://testserver/") as client:
            yield client 

def stream_download(self, chunk_size: Optional[int] = None, callback: Optional[Callable] = None) -> AsyncIterator[bytes]:
        """Generator for streaming request body data.
        """
        chunk_size = chunk_size or CONTENT_CHUNK_SIZE
        async def async_gen(resp):
            while True:
                chunk = await resp.content.read(chunk_size)
                if not chunk:
                    break
                callback(chunk, resp)
        return async_gen(self.internal_response) 

def stream_download(self, chunk_size: Optional[int] = None, callback: Optional[Callable] = None) -> AsyncIterator[bytes]:
        """Generator for streaming request body data.

        Should be implemented by sub-classes if streaming download
        is supported.

        :param callback: Custom callback for monitoring progress.
        :param int chunk_size:
        """
        pass 

def stream_download(self, chunk_size: Optional[int] = None, callback: Optional[Callable] = None) -> AsyncIteratorType[bytes]:
            """Generator for streaming request body data.

            :param callback: Custom callback for monitoring progress.
            :param int chunk_size:
            """
            return TrioStreamDownloadGenerator(
                self.internal_response,
                callback,
                chunk_size
            ) 

def streaming_watch(
        *,
        settings: configuration.OperatorSettings,
        resource: resources.Resource,
        namespace: Optional[str],
        freeze_mode: Optional[primitives.Toggle] = None,
) -> AsyncIterator[bodies.RawEvent]:

    # Prevent both watching and listing while the freeze mode is on, until it is off.
    # Specifically, the watch-stream closes its connection once the freeze mode is on,
    # so the while-true & for-event-in-stream cycles exit, and this coroutine is started
    # again by the `infinite_stream()` (the watcher timeout is swallowed by the freeze time).
    if freeze_mode is not None and freeze_mode.is_on():
        logger.debug("Freezing the watch-stream for %r", resource)
        await freeze_mode.wait_for_off()
        logger.debug("Resuming the watch-stream for %r", resource)

    # A stop-feature is a client-specific way of terminating the streaming HTTPS connection
    # when a freeze-mode is turned on. The low-level API call attaches its `response.close()`
    # to the future's callbacks, and a background task triggers it when the mode is turned on.
    freeze_waiter: asyncio_Future
    if freeze_mode is not None:
        freeze_waiter = asyncio.create_task(freeze_mode.wait_for_on())
    else:
        freeze_waiter = asyncio.Future()  # a dummy just ot have it

    try:
        stream = continuous_watch(
            settings=settings,
            resource=resource, namespace=namespace,
            freeze_waiter=freeze_waiter,
        )
        async for raw_event in stream:
            yield raw_event
    finally:
        with contextlib.suppress(asyncio.CancelledError):
            freeze_waiter.cancel()
            await freeze_waiter 

def stream_download(self, chunk_size: Optional[int] = None, callback: Optional[Callable] = None) -> AsyncIteratorType[bytes]:
        """Generator for streaming request body data.

        :param callback: Custom callback for monitoring progress.
        :param int chunk_size:
        """
        return StreamDownloadGenerator(
            self.internal_response,
            callback,
            chunk_size
        )


# Trio support 

def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\
            -> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
        if start_blob_num >= len(self.descriptor.blobs[:-1]):
            raise IndexError(start_blob_num)
        for i, blob_info in enumerate(self.descriptor.blobs[start_blob_num:-1]):
            assert i + start_blob_num == blob_info.blob_num
            if connection_id == self.STREAMING_ID:
                decrypted = await self.downloader.cached_read_blob(blob_info)
            else:
                decrypted = await self.downloader.read_blob(blob_info, connection_id)
            yield (blob_info, decrypted) 

def __aiter__(self) -> AsyncIterator:
        return self 

def __aiter__(self) -> AsyncIterator:
        """Iterate asynchronously.

        Transforms the async source to an async iterable. The source
        will await for the iterator to pick up the value before
        continuing to avoid queuing values.
        """

        obv = AsyncIteratorObserver()
        await self._source.__asubscribe__(obv)
        return obv 

def _rtm(self, url: str) -> AsyncIterator[str]:
        yield ""
        raise NotImplementedError