Python aiohttp.ClientResponse() Examples

def test_iter_lines_generator():
    """Test that lines are split correctly."""
    async def mock_iter_content(n):
        for chunk in [b'1\r\n2\r\n', b'3\r', b'\n4', b'\r\n5']:
            yield chunk

    response = ClientResponse(
        'get', ST_URL,
        request_info=Mock(),
        writer=Mock(),
        continue100=None,
        timer=TimerNoop(),
        traces=[],
        loop=Mock(),
        session=Mock(),
    )
    response._headers = {'Content-Type': 'application/json;charset=utf-8'}
    with patch.object(response, 'content', Mock(iter_chunked=mock_iter_content)):
        result = [
            line async for line in _iter_lines_generator(
                response=response, decode_unicode=True
            )
        ]
        assert result == ['1', '2', '3', '4', '5'] 

def validate(self, response: aiohttp.ClientResponse, checker_result: CheckerResult) -> bool:
        if response.status != 200:
            return False

        json_result = await response.json()
        if 'ip' in json_result:
            checker_result.ipv4 = json_result['ip']
        if 'city' in json_result:
            checker_result.city = json_result['city']
        if 'region' in json_result:
            checker_result.region = json_result['region']
        if 'country' in json_result:
            checker_result.country_code = json_result['country']
        if 'loc' in json_result:
            checker_result.location_coordinates = tuple(float(x) for x in json_result['loc'].split(','))
        if 'org' in json_result:
            checker_result.organization_name = json_result['org']

        return True 

def get_active_exchange_markets(cls,
                                          api_endpoint: str = "https://rest.bamboorelay.com/",
                                          api_prefix: str = "main/0x") -> pd.DataFrame:
        """
        Returned data frame should have trading_pair as index and include usd volume, baseAsset and quoteAsset
        """
        client: aiohttp.ClientSession = cls.http_client()
        async with client.get(f"{api_endpoint}{api_prefix}/markets?perPage=1000&include=ticker,stats") as response:
            response: aiohttp.ClientResponse = response
            if response.status != 200:
                raise IOError(f"Error fetching active Bamboo Relay markets. HTTP status is {response.status}.")
            data = await response.json()
            data: List[Dict[str, any]] = [
                {**item, **{"baseAsset": item["id"].split("-")[0], "quoteAsset": item["id"].split("-")[1]}}
                for item in data
            ]
            all_markets: pd.DataFrame = pd.DataFrame.from_records(data=data, index="id")

            quote_volume: List[float] = []
            for row in all_markets.itertuples():
                base_volume: float = float(row.stats["volume24Hour"])
                quote_volume.append(base_volume)

            all_markets.loc[:, "volume"] = quote_volume
            return all_markets.sort_values("USDVolume", ascending=False) 

def get_active_exchange_markets(cls) -> pd.DataFrame:
        """
        Returned data frame should have trading pair as index and include usd volume, baseAsset and quoteAsset
        """
        client: aiohttp.ClientSession = cls.http_client()
        async with client.get(f"{MARKETS_URL}?include=ticker,stats") as response:
            response: aiohttp.ClientResponse = response
            if response.status != 200:
                raise IOError(f"Error fetching active Radar Relay markets. HTTP status is {response.status}.")
            data = await response.json()
            data: List[Dict[str, any]] = [
                {**item, **{"baseAsset": item["id"].split("-")[0], "quoteAsset": item["id"].split("-")[1]}}
                for item in data
            ]
            all_markets: pd.DataFrame = pd.DataFrame.from_records(data=data, index="id")

            quote_volume: List[float] = []
            for row in all_markets.itertuples():
                base_volume: float = float(row.stats["volume24Hour"])
                quote_volume.append(base_volume)

            all_markets.loc[:, "volume"] = quote_volume
            return all_markets.sort_values("USDVolume", ascending=False) 

def get_snapshot(client: aiohttp.ClientSession, trading_pair: str, limit: int = 1000) -> Dict[str, Any]:
        original_trading_pair: str = trading_pair
        params: Dict[str, str] = {"count": str(limit), "pair": trading_pair} if limit != 0 else {"pair": trading_pair}
        async with client.get(SNAPSHOT_REST_URL, params=params) as response:
            response: aiohttp.ClientResponse = response
            if response.status != 200:
                raise IOError(f"Error fetching Kraken market snapshot for {original_trading_pair}. "
                              f"HTTP status is {response.status}.")
            response_json = await response.json()
            if len(response_json["error"]) > 0:
                raise IOError(f"Error fetching Kraken market snapshot for {original_trading_pair}. "
                              f"Error is {response_json['error']}.")
            data: Dict[str, Any] = next(iter(response_json["result"].values()))
            data = {"trading_pair": trading_pair, **data}
            data["latest_update"] = max([*map(lambda x: x[2], data["bids"] + data["asks"])], default=0.)

            # Need to add the symbol into the snapshot message for the Kafka message queue.
            # Because otherwise, there'd be no way for the receiver to know which market the
            # snapshot belongs to.

            return data 

def _query_qna_service(
        self, turn_context: TurnContext, options: QnAMakerOptions
    ) -> QueryResults:
        url = f"{ self._endpoint.host }/knowledgebases/{ self._endpoint.knowledge_base_id }/generateAnswer"

        question = GenerateAnswerRequestBody(
            question=turn_context.activity.text,
            top=options.top,
            score_threshold=options.score_threshold,
            strict_filters=options.strict_filters,
            context=options.context,
            qna_id=options.qna_id,
            is_test=options.is_test,
            ranker_type=options.ranker_type,
        )

        http_request_helper = HttpRequestUtils(self._http_client)

        response: ClientResponse = await http_request_helper.execute_http_request(
            url, question, self._endpoint, options.timeout
        )

        result: QueryResults = await self._format_qna_result(response, options)

        return result 

def fetch(self, url, **kwargs) -> aiohttp.ClientResponse:
        headers = {"User-Agent": get_user_agent()}
        async with aiohttp.ClientSession(
            conn_timeout=self.config["response_timeout"],
            read_timeout=self.config["response_timeout"],
        ) as session:
            try:
                async with session.get(
                    urljoin(self.host, url), headers=headers, **kwargs
                ) as response:
                    await response.text()
                    return response
            except aiohttp.ClientConnectionError:
                raise exceptions.SlaveDoesNotExist(
                    f"Unable to connect to the slave at {self.host}"
                ) 

def _read_content(self, response: aiohttp.ClientResponse):
        content_length = response.headers.get('Content-Length')
        if content_length:
            content_length = int(content_length)
            if content_length > self.max_content_length:
                msg = 'content length {} larger than limit {}'.format(
                    content_length, self.max_content_length)
                raise ContentTooLargeError(msg)
        content_length = 0
        content = bytearray()
        async for chunk in response.content.iter_chunked(8 * 1024):
            content_length += len(chunk)
            if content_length > self.max_content_length:
                msg = 'content length larger than limit {}'.format(
                    self.max_content_length)
                raise ContentTooLargeError(msg)
            content.extend(chunk)
        return content 

def request(self, method: str, url: str,
                      **kwargs: Any
                      ) -> Tuple[aiohttp.ClientResponse, Union[str, dict]]:
        try:
            params = kwargs['params']
            if isinstance(params, dict):
                kwargs['params'] = {k: (str(v).lower() if isinstance(v, bool)
                                        else v) for k, v in params.items()}
            else:
                kwargs['params'] = [(k, (str(v).lower() if isinstance(v, bool)
                                         else v)) for k, v in params]
        except KeyError:
            pass

        pre_time = time.time()
        async with self.__session.request(method, url, **kwargs) as r:
            log.debug('{0} {1} has returned {2.status} in {3:.2f}s'.format(
                method,
                url,
                r,
                time.time() - pre_time
            ))

            data = await self.json_or_text(r)
            return r, data 

def _read(
            self, url, etag=None, last_modified=None, referer=None,
            headers=None, ignore_content=False
    ) -> aiohttp.ClientResponse:
        headers = self._prepare_headers(
            etag=etag,
            last_modified=last_modified,
            referer=referer,
            headers=headers,
        )
        await self._async_init()
        if not self.allow_private_address:
            await self.check_private_address(url)
        async with self.session.get(url, headers=headers) as response:
            content = None
            if not is_ok_status(response.status) or not ignore_content:
                content = await self._read_content(response)
            if not is_ok_status(response.status):
                return response.headers, content, url, response.status
            self.check_content_type(response)
        return response.headers, content, str(response.url), response.status 

def setUp(self):
        self.main_page_path = generate_unique_path()
        os.makedirs(self.main_page_path)
        self.dorks = dict(response={'dorks': "test_dorks"})
        self.loop = asyncio.new_event_loop()
        aiohttp.ClientSession.get = AsyncMock(
            return_value=aiohttp.ClientResponse(
                url=yarl.URL("http://www.example.com"),
                method="GET",
                writer=None,
                continue100=1,
                timer=None,
                request_info=None,
                traces=None,
                loop=self.loop,
                session=None))
        no_dorks = True
        tanner = "tanner.mushmush.org"
        self.handler = HtmlHandler(no_dorks, tanner)
        self.data = None 

def post(self, path, *path_args, data=None):
    """Sends a POST request to a given API endpoint.

    This is a low-level function that returns a raw HTTP response, no error
    checking nor response parsing is performed. See :func:`post_object` for
    a higher-level function.

    :param path: Path to API endpoint, can contain format placeholders {}.
    :param path_args: A variable number of arguments that are put into any
      placeholders used in path.
    :param data: Data sent in the request body.
    :type path: str
    :type data: A string or bytes
    :returns: An instance of :class:`ClientResponse`.
    """
    return _make_sync(self.post_async(path, *path_args, data=data)) 

def get_error_async(self, response):
    """Given a :class:`ClientResponse` returns a :class:`APIError`

    This function checks if the response from the VirusTotal backend was an
    error and returns the appropiate :class:`APIError` or None if no error
    occurred.

    :param response: A :class:`ClientResponse` instance.
    :returns: An instance of :class:`APIError` or None.
    """
    if response.status == 200:
      return None
    if response.status >= 400 and response.status <= 499:
      if response.content_type == 'application/json':
        json_response = await response.json_async()
        error = json_response.get('error')
        if error:
          return APIError.from_dict(error)
      return APIError('ClientError', await response.text_async())
    return APIError('ServerError', await response.text_async()) 

def get(self, path, *path_args, params=None):
    """Sends a GET request to a given API endpoint.

    This is a low-level function that returns a raw HTTP response, no error
    checking nor response parsing is performed. See :func:`get_json`,
    :func:`get_data` and :func:`get_object` for higher-level functions.

    :param path: Path to API endpoint, can contain format placeholders {}.
    :param path_args: A variable number of arguments that are put into any
      placeholders used in path.
    :param params: Parameters sent in the request.
    :type path: str
    :type params: dict
    :returns: An instance of :class:`ClientResponse`.
    """
    return _make_sync(self.get_async(path, *path_args, params=params)) 

def test_event_result(self):
        aiohttp.ClientResponse.json = AsyncMock(
            return_value=dict(
                detection={
                    'type': 1},
                sess_uuid="test_uuid"))

        async def test():
            self.result = await self.handler.submit_data(self.data)

        self.loop.run_until_complete(test())
        self.assertEqual(
            self.result,
            dict(
                detection={
                    'type': 1},
                sess_uuid="test_uuid")) 

def _wrap_response(
        self,
        response: aiohttp.ClientResponse,
        url: str,
        **kwargs: Union[int, Optional[str]],
    ) -> Dict[str, Any]:
        """Parses the response as json, then runs check_response and
        add_jikan_metadata
        """
        json_response: Dict[str, Any] = {}
        try:
            json_response = await response.json()
            if not isinstance(json_response, dict):
                json_response = {"data": json_response}
        except (json.decoder.JSONDecodeError, simplejson.JSONDecodeError):
            json_response = {"error": await response.text()}
        if response.status >= 400:
            raise APIException(response.status, json_response, **kwargs)
        return utils.add_jikan_metadata(response, json_response, url) 

def get(
        self, path: str, required_serial: Optional[int], **kw: Any
    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
        logger.debug(f"Getting {path} (serial {required_serial})")
        if not path.startswith(("https://", "http://")):
            path = self.url + path

        async with self.session.get(path, **kw) as r:
            got_serial = (
                int(r.headers[PYPI_SERIAL_HEADER])
                if PYPI_SERIAL_HEADER in r.headers
                else None
            )
            await self.check_for_stale_cache(path, required_serial, got_serial)
            yield r

    # TODO: Add storage backend support / refactor - #554 

def get_snapshot(client: aiohttp.ClientSession, trading_pair: str) -> Dict[str, Any]:
        # when type is set to "step0", the default value of "depth" is 150
        async with client.get("/mockSnapshot") as response:
            response: aiohttp.ClientResponse = response
            if response.status != 200:
                raise IOError(f"Error fetching market snapshot for {trading_pair}. "
                              f"HTTP status is {response.status}.")
            parsed_response = await response.json()
            return parsed_response 

def delete(self, path, *path_args):
    """Sends a DELETE request to a given API endpoint.

    :param path: Path to API endpoint, can contain format placeholders {}.
    :param path_args: A variable number of arguments that are put into any
      placeholders used in path.
    :type path: str
    :returns: An instance of :class:`ClientResponse`.
    """
    return _make_sync(self.delete_async(path, *path_args)) 

def _handle_error(self, response):
        """Handle http request errors.
        
        This method is called whenever an API call returns an HTTP status code
        of 400 or higher.
        
        :param aiohttp.ClientResponse response: the response returned by the
            aiohttp request call.
        """
        if response.content_type.lower() == 'text/html':
            msg = await response.text()
        else:
            msg = (await response.json())['message']
        msg += ' [{}]'.format(response.status)
        raise APIRequestError(msg, response) 

def test_iter_content_generator():
    """Test CRLF -> LF newline conversion."""
    async def mock_iter_content(n):
        for chunk in [b'1\r\n2\r\n', b'3\r', b'\n4', b'\r\n5']:
            yield chunk

    response = ClientResponse(
        'get', ST_URL,
        request_info=Mock(),
        writer=Mock(),
        continue100=None,
        timer=TimerNoop(),
        traces=[],
        loop=Mock(),
        session=Mock(),
    )
    response._headers = {'Content-Type': 'application/json;charset=utf-8'}
    with patch.object(response, 'content', Mock(iter_chunked=mock_iter_content)):
        result = [
            line async for line in _iter_content_generator(
                response=response, decode_unicode=True
            )
        ]
        assert result == ['1\n2\n', '3', '\n4', '\n5']

        result = [
            line async for line in _iter_content_generator(
                response=response, decode_unicode=False
            )
        ]
        assert result == [b'1\r\n2\r\n', b'3\r', b'\n4', b'\r\n5'] 

def get_all_token_info(cls,
                                 api_endpoint: str = "https://rest.bamboorelay.com/",
                                 api_prefix: str = "") -> Dict[str, any]:
        """
        Returns all token information
        """
        client: aiohttp.ClientSession = cls.http_client()
        async with client.get(f"{api_endpoint}{api_prefix}/tokens?perPage=1000") as response:
            response: aiohttp.ClientResponse = response
            if response.status != 200:
                raise IOError(f"Error fetching token info. HTTP status is {response.status}.")
            data = await response.json()
            return {d["address"]: d for d in data} 

def get_all_token_info(cls) -> Dict[str, any]:
        """
        Returns all token information
        """
        client: aiohttp.ClientSession = cls.http_client()
        async with client.get(TOKENS_URL) as response:
            response: aiohttp.ClientResponse = response
            if response.status != 200:
                raise IOError(f"Error fetching token info. HTTP status is {response.status}.")
            data = await response.json()
            return {d["address"]: d for d in data} 

def _raise_for_status(response):
    """Raise an appropriate error for a given response.

    Arguments:
      response (:py:class:`aiohttp.ClientResponse`): The API response.

    Raises:
      :py:class:`aiohttp.ClientResponseError`: The appropriate
        error for the response's status.
    """

    if 400 <= response.status:
        reason = response.reason

        spacetrack_error_msg = None

        try:
            json = await response.json()
            if isinstance(json, Mapping):
                spacetrack_error_msg = json['error']
        except (ValueError, KeyError, aiohttp.ClientResponseError):
            pass

        if not spacetrack_error_msg:
            spacetrack_error_msg = await response.text()

        if spacetrack_error_msg:
            reason += '\nSpace-Track response:\n' + spacetrack_error_msg

        raise aiohttp.ClientResponseError(
            response.request_info,
            response.history,
            status=response.status,
            message=reason,
            headers=response.headers) 

def _get(self, *args, **kwargs):
        """
        Wrapper for :func:`requests.get` that injects the connection's
        Threema ID and its secret.

        Return a :class:`aiohttp.ClientResponse` instance.
        """
        kwargs.setdefault('params', {})
        kwargs['params'].setdefault('from', self.id)
        kwargs['params'].setdefault('secret', self.secret)
        return (yield from self._session.get(*args, **kwargs)) 

def async_validate_response(web_response: ClientResponse) -> Dict[str, Any]:
    """Validate response to API command is as expected and return response."""
    if HTTP_OK != web_response.status:
        raise Exception(
            "Device is unreachable? Status code: {0}".format(web_response.status)
        )

    try:
        data = json.loads(await web_response.text())
    except Exception:
        raise Exception("Failed to parse response: {0}".format(web_response.content))

    status_obj = dict_get_case_insensitive(data, "status")

    if not status_obj:
        raise Exception("Unknown response")

    result_status = dict_get_case_insensitive(status_obj, "result")

    if result_status and result_status.lower() == STATUS_INVALID_PARAMETER:
        raise Exception("invalid value specified")
    elif not result_status or result_status.lower() != STATUS_SUCCESS:
        raise Exception(
            "unexpected status {0}: {1}".format(
                result_status, dict_get_case_insensitive(status_obj, "detail")
            )
        )

    return data 

def get_snapshot(client: aiohttp.ClientSession, trading_pair: str) -> Dict[str, any]:
        async with client.get(f"{REST_BASE_URL}/markets/{trading_pair}/book") as response:
            response: aiohttp.ClientResponse = response
            if response.status != 200:
                raise IOError(f"Error fetching Radar Relay market snapshot for {trading_pair}. "
                              f"HTTP status is {response.status}.")
            return await response.json() 

def validate(self, response: aiohttp.ClientResponse, checker_result: CheckerResult):
        '''
        We have already done the request and it was successful,
        Google returned something(maybe good response, maybe captcha, we don't care)
        '''
        return True 

def validate(
        self, response: aiohttp.ClientResponse, checker_result: CheckerResult
    ) -> bool:
        """
        Implement this method. It will get response from url with http method you provided in constructor

        :param response: aiohttp response
        :param checker_result: fill this structure with information like ip address
        :return: whether proxy is working or not
        """
        raise NotImplemented() 

def get_listen_key(self):
        async with aiohttp.ClientSession() as client:
            async with client.post(f"{BINANCE_API_ENDPOINT}{BINANCE_USER_STREAM_ENDPOINT}",
                                   headers={"X-MBX-APIKEY": self._binance_client.API_KEY}) as response:
                response: aiohttp.ClientResponse = response
                if response.status != 200:
                    raise IOError(f"Error fetching Binance user stream listen key. HTTP status is {response.status}.")
                data: Dict[str, str] = await response.json()
                return data["listenKey"]