Python asyncio.LimitOverrunError() Examples

def run(self):
        await self._client_connected()
        while not self.__writer.is_closing():
            try:
                data = await self.__reader.readuntil(
                    separator=Spheniscidae.Delimiter)
                if data:
                    await self.__data_received(data)
                else:
                    self.__writer.close()
                await self.__writer.drain()
            except IncompleteReadError:
                self.__writer.close()
            except CancelledError:
                self.__writer.close()
            except ConnectionResetError:
                self.__writer.close()
            except LimitOverrunError:
                self.__writer.close()
            except BaseException as e:
                self.logger.exception(e.__traceback__)

        await self._client_disconnected() 

def test_readuntil_limit_found_sep(self):
        stream = asyncio.StreamReader(loop=self.loop, limit=3)
        stream.feed_data(b'some dataAA')

        with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                    'not found') as cm:
            self.loop.run_until_complete(stream.readuntil(b'AAA'))

        self.assertEqual(b'some dataAA', stream._buffer)

        stream.feed_data(b'A')
        with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                    'is found') as cm:
            self.loop.run_until_complete(stream.readuntil(b'AAA'))

        self.assertEqual(b'some dataAAA', stream._buffer) 

def test_readuntil_limit_found_sep(self):
        stream = asyncio.StreamReader(loop=self.loop, limit=3)
        stream.feed_data(b'some dataAA')

        with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                    'not found') as cm:
            self.loop.run_until_complete(stream.readuntil(b'AAA'))

        self.assertEqual(b'some dataAA', stream._buffer)

        stream.feed_data(b'A')
        with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                    'is found') as cm:
            self.loop.run_until_complete(stream.readuntil(b'AAA'))

        self.assertEqual(b'some dataAAA', stream._buffer) 

def readuntil(self, separator=b"\n"):
        # Re-implement `readuntil` to work around self._limit.
        # The limit is still useful to prevent the internal buffer
        # from growing too large when it's not necessary, but it
        # needs to be disabled when the user code is purposely
        # reading from stdin.
        while True:
            try:
                return await super().readuntil(separator)
            except asyncio.LimitOverrunError as e:
                if self._buffer.startswith(separator, e.consumed):
                    chunk = self._buffer[: e.consumed + len(separator)]
                    del self._buffer[: e.consumed + len(separator)]
                    self._maybe_resume_transport()
                    return bytes(chunk)
                await self._wait_for_data("readuntil") 

def test_readuntil_limit_found_sep(self):
        stream = asyncio.StreamReader(loop=self.loop, limit=3)
        stream.feed_data(b'some dataAA')

        with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                    'not found') as cm:
            self.loop.run_until_complete(stream.readuntil(b'AAA'))

        self.assertEqual(b'some dataAA', stream._buffer)

        stream.feed_data(b'A')
        with self.assertRaisesRegex(asyncio.LimitOverrunError,
                                    'is found') as cm:
            self.loop.run_until_complete(stream.readuntil(b'AAA'))

        self.assertEqual(b'some dataAAA', stream._buffer) 

def read_stream(self, stream):
        while self.is_active:
            try:
                line = await stream.readline()
            except (asyncio.LimitOverrunError, ValueError):
                continue
            if line:
                line = line.decode("utf-8")[:-1]
                print(f"[Cluster {self.id}] {line}")
            else:
                break 

def async_run_command(self, command, first_try=True):
        """Run a command through a Telnet connection.
        Connect to the Telnet server if not currently connected, otherwise
        use the existing connection.
        """
        await self.async_connect()
        try:
            with (await self._io_lock):
                self._writer.write('{}\n'.format(
                    "%s && %s" % (
                        _PATH_EXPORT_COMMAND, command)).encode('ascii'))
                data = ((await asyncio.wait_for(self._reader.readuntil(
                    self._prompt_string), 9)).split(b'\n')[1:-1])

        except (BrokenPipeError, LimitOverrunError):
            if first_try:
                return await self.async_run_command(command, False)
            else:
                _LOGGER.warning("connection is lost to host.")
                return[]
        except TimeoutError:
            _LOGGER.error("Host timeout.")
            return []
        finally:
            self._writer.close()

        return [line.decode('utf-8') for line in data] 

def test_LimitOverrunError_pickleable(self):
        e = asyncio.LimitOverrunError('message', 10)
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            with self.subTest(pickle_protocol=proto):
                e2 = pickle.loads(pickle.dumps(e, protocol=proto))
                self.assertEqual(str(e), str(e2))
                self.assertEqual(e.consumed, e2.consumed) 

def connection_loop(execute_rpc: Callable[[Any], Any],
                          reader: asyncio.StreamReader,
                          writer: asyncio.StreamWriter) -> None:
    # TODO: we should look into using an io.StrinIO here for more efficient
    # writing to the end of the string.
    raw_request = ''
    while True:
        request_bytes = b''
        try:
            request_bytes = await reader.readuntil(b'}')
        except asyncio.LimitOverrunError as e:
            logger.info("Client request was too long. Erasing buffer and restarting...")
            request_bytes = await reader.read(e.consumed)
            await write_error(
                writer,
                f"reached limit: {e.consumed} bytes, starting with '{request_bytes[:20]!r}'",
            )
            continue

        raw_request += request_bytes.decode()

        bad_prefix, raw_request = strip_non_json_prefix(raw_request)
        if bad_prefix:
            logger.info("Client started request with non json data: %r", bad_prefix)
            await write_error(writer, 'Cannot parse json: ' + bad_prefix)

        try:
            request = json.loads(raw_request)
        except json.JSONDecodeError:
            # invalid json request, keep reading data until a valid json is formed
            logger.debug("Invalid JSON, waiting for rest of message: %r", raw_request)
            continue

        # reset the buffer for the next message
        raw_request = ''

        if not request:
            logger.debug("Client sent empty request")
            await write_error(writer, 'Invalid Request: empty')
            continue

        try:
            result = await execute_rpc(request)
        except Exception as e:
            logger.exception("Unrecognized exception while executing RPC")
            await write_error(writer, "unknown failure: " + str(e))
        else:
            if not result.endswith(NEW_LINE):
                result += NEW_LINE

            writer.write(result.encode())

        await writer.drain() 

def stream(self, chunk_size: int=1*1024*1024, chunk_timeout: int=10,
                     complete_timeout: int=300):
        """

        :param complete_timeout:
        :param chunk_timeout:
        :param chunk_size:
        :return:
        """
        if self._parser_status not in (ResponseStatus.PENDING_HEADERS, ResponseStatus.PENDING_BODY):
            raise StreamAlreadyConsumed
        if self._parser_status == ResponseStatus.PENDING_HEADERS:
            await wait_for(self.receive_headers(), chunk_timeout)

        try:
            length = int(self.headers['Content-Length'])
            remaining = length
            while remaining:
                bytes_to_read = min(remaining, chunk_size)
                task = self._connection.read_exactly(bytes_to_read)
                start_time = time.time()
                self._parser.feed(await wait_for(task, min(chunk_timeout, complete_timeout)))
                complete_timeout -= time.time() - start_time
                remaining -= bytes_to_read
                yield bytes(self._content)
                self._content = bytearray()
        except KeyError:
            if self._headers.get('Transfer-Encoding') == 'chunked':
                self._parser_status = ResponseStatus.CHUNKED_TRANSFER
                while self._parser_status == ResponseStatus.CHUNKED_TRANSFER:
                    task = self._connection.read_until(b'\r\n')
                    start_time = time.time()
                    try:
                        self._parser.feed(await wait_for(task, min(chunk_timeout, complete_timeout)))
                    except asyncio.LimitOverrunError as error:
                        self._parser.feed(await self._connection.read_exactly(error.consumed))
                    complete_timeout -= time.time() - start_time
                    while len(self._content) >= chunk_size:
                        yield self._content[:chunk_size]
                        self._content = self._content[chunk_size:]
                while len(self._content) > 0:
                    yield self._content[:chunk_size]
                    self._content = self._content[chunk_size:]
            else:
                raise Exception('Invalid response.')
        except ValueError:
            raise Exception('Invalid content-length header.')