Python aiohttp.ClientError() Examples

def fetch(self, url, max_redirect):
        tries = 0
        exception = None
        while tries < self.max_tries:
            try:
                response = await self.session.get(
                    url, allow_redirects=False)
                break
            except aiohttp.ClientError as client_error:
                exception = client_error

            tries += 1
        else:
            return

        try:
            next_url = await self.parse_link(response)
            print('{} has finished'.format(url))
            if next_url is not None:
                self.add_url(next_url, max_redirect)
        finally:
            response.release() 

def get_highscores(world, category=Category.EXPERIENCE, vocation=VocationFilter.ALL, *, tries=5) \
        -> Optional[Highscores]:
    """Gets all the highscores entries of a world, category and vocation."""
    # TODO: Add caching
    if tries == 0:
        raise errors.NetworkError(f"get_highscores({world},{category},{vocation})")

    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(Highscores.get_url_tibiadata(world, category, vocation)) as resp:
                content = await resp.text()
                highscores = Highscores.from_tibiadata(content, vocation)
    except (aiohttp.ClientError, asyncio.TimeoutError, tibiapy.TibiapyException):
        await asyncio.sleep(config.network_retry_delay)
        return await get_highscores(world, category, vocation, tries=tries - 1)

    return highscores 

def get(self, url: str, params: Dict[str, str] = None) -> Response:
        """Perform HTTP GET request.

        :param url: the request url
        :param params: the request params
        :return: the response from server
        :raise: :exc:`ConnectionError <stellar_sdk.exceptions.ConnectionError>`
        """
        try:
            response = await self._session.get(url, params=params)
            return Response(
                status_code=response.status,
                text=await response.text(),
                headers=dict(response.headers),
                url=str(response.url),
            )
        except aiohttp.ClientError as e:  # TODO: need more research
            raise ConnectionError(e) 

def aiohttp_repeat(func=None, *, count: int = 4):
    if func is None:
        return partial(func, count=count)

    async def wrapper(*args: Any, **kwargs: Any) -> Optional[Any]:
        for pause in range(1, count + 1):
            try:
                return await func(*args, **kwargs)
            except ClientError:
                if pause == count:
                    raise
                logger.debug('aiohttp payload error, repeating...', exc_info=True)
                sleep(pause)
        raise RuntimeError('unreachable')

    wrapper = update_wrapper(wrapper=wrapper, wrapped=func)
    return wrapper 

def wait_til_server_is_running(
    endpoint, max_retries=30, sleep_between_retries=1
) -> bool:
    """Try to reach the server, retry a couple of times and sleep in between."""

    while max_retries:
        try:
            r = await retrieve_status(endpoint)
            logger.info(f"Reached core: {r}")
            if not r.get("is_ready"):
                # server did not finish loading the agent yet
                # in this case, we need to wait till the model trained
                # so we might be sleeping for a while...
                await asyncio.sleep(sleep_between_retries)
                continue
            else:
                # server is ready to go
                return True
        except ClientError:
            max_retries -= 1
            if max_retries:
                await asyncio.sleep(sleep_between_retries)

    return False 

def async_update(self):
    try:
      auth = aiohttp.BasicAuth(self.username, self.password)
      with async_timeout.timeout(TIMEOUT, loop=self.hass.loop):
        response = await self.websession.get(ENDPOINT, auth=auth)
        data = await response.json(content_type=None)
        if len(data) > 0:
          _LOGGER.debug("Updating sensor: {}".format(data))
          entry = data[0]
          self._meal = entry['meal']
          self.extract_deilver_date(entry['deliveryDate'])
        else:
          _LOGGER.debug("No data to update: {}".format(data))
          self._deliver_from = None
          self._deliver_to = None
          self._time_left = None
          self._meal = None
    except (asyncio.TimeoutError, aiohttp.ClientError, IndexError) as error:
      _LOGGER.error("Failed getting devices: %s", error) 

def get_config_via_legacy_route(bf_url, project_id):
    from rasa.utils.endpoints import EndpointConfig
    import aiohttp

    response = {}
    base_url = f"{bf_url}/project/{project_id}"
    for endpoint in ["credentials", "endpoints"]:
        server = EndpointConfig(url=f"{base_url}/{endpoint}")
        async with server.session() as session:
            params = server.combine_parameters()
            url = server.url

            @auto_retry
            async def load():
                try:
                    return await session.request(
                        "GET", url, timeout=DEFAULT_REQUEST_TIMEOUT, params=params
                    )
                except aiohttp.ClientError:
                    return None

            data = await load()
            response[endpoint] = await data.json()
    return response 

def async_update(self):
    try:
      from bs4 import BeautifulSoup
      with async_timeout.timeout(TIMEOUT, loop=self.hass.loop):
        response = await self.websession.get(ENDPOINT, params={ "identityNumber": self.identity_id, "cityCardNumber": self.city_card_id })
        data = await response.text()
        #_LOGGER.debug(data)
        raw_data = BeautifulSoup(data, 'html.parser')
        self.extract_date(raw_data)

        if self.days() == 0:
          self._state = STATE_OFF
        else:
          self._state = STATE_ON
    except (asyncio.TimeoutError, aiohttp.ClientError) as error:
      _LOGGER.error("Failed getting kkm information: %s", error) 

def async_check_http_oauth(self, triggered=None):
        _LOGGER.debug("[%s] check accessibility from local", LOGGER_NAME)
        try:
            if self._retry_remove is not None:
                self._retry_remove()
                self._retry_remove = None

            session = async_get_clientsession(self._hass, verify_ssl=False)
            with async_timeout.timeout(5, loop= self._hass.loop):
                response = await session.get(self._ha_url + '/havcs/auth/authorize')
            if response.status == 401:
                _LOGGER.debug("[%s][check] access success: url = %s, status = %s", LOGGER_NAME, self._ha_url + '/havcs/auth/authorize', response.status)
        except (asyncio.TimeoutError, aiohttp.ClientError):
            _LOGGER.debug("[%s][check] retry check after 15s", LOGGER_NAME)
            self._retry_times -= 1
            if(self._retry_times > 0):
                self._retry_remove = async_track_time_interval(
                    self._hass, self.async_check_http_oauth, timedelta(seconds=15)
                )
            else:
                _LOGGER.error("[%s][check] can not access http, check `ha_url` in configuration.yml", LOGGER_NAME)
        except Exception:
            _LOGGER.exception("[%s][check] unexpected error occur", LOGGER_NAME)
            raise 

def fetch(self, url):
        """Fetch request."""
        error_msg = None
        try:
            async with aiohttp.ClientSession() as session:
                body = await self.fetch_with_session(session, url)
        except asyncio.TimeoutError:
            error_msg = 'Request timed out'
            raise ClashRoyaleAPIError(message=error_msg)
        except aiohttp.ServerDisconnectedError as err:
            error_msg = 'Server disconnected error: {}'.format(err)
            raise ClashRoyaleAPIError(message=error_msg)
        except (aiohttp.ClientError, ValueError) as err:
            error_msg = 'Request connection error: {}'.format(err)
            raise ClashRoyaleAPIError(message=error_msg)
        except json.JSONDecodeError:
            error_msg = "Non JSON returned"
            raise ClashRoyaleAPIError(message=error_msg)
        else:
            return body
        finally:
            if error_msg is not None:
                raise ClashRoyaleAPIError(message=error_msg) 

def get_world(name, *, tries=5) -> Optional[World]:
    name = name.strip().title()
    if tries == 0:
        raise errors.NetworkError(f"get_world({name})")
    try:
        world = CACHE_WORLDS[name]
        return world
    except KeyError:
        pass
    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(World.get_url_tibiadata(name)) as resp:
                content = await resp.text(encoding='ISO-8859-1')
                world = World.from_tibiadata(content)
    except (aiohttp.ClientError, asyncio.TimeoutError, tibiapy.TibiapyException):
        await asyncio.sleep(config.network_retry_delay)
        return await get_world(name, tries=tries - 1)
    CACHE_WORLDS[name] = world
    return world 

def wait_til_server_is_running(endpoint,
                                     max_retries=30,
                                     sleep_between_retries=1):
    """Try to reach the server, retry a couple of times and sleep in between."""

    while max_retries:
        try:
            r = await retrieve_status(endpoint)
            logger.info("Reached core: {}".format(r))
            if not r.get("is_ready"):
                # server did not finish loading the agent yet
                # in this case, we need to wait till the model trained
                # so we might be sleeping for a while...
                await asyncio.sleep(sleep_between_retries)
                continue
            else:
                # server is ready to go
                return True
        except ClientError:
            max_retries -= 1
            if max_retries:
                await asyncio.sleep(sleep_between_retries)

    return False 

def get_response(session, url, headers):
    try:
        response = await session.get(url, headers=headers)
    except (OSError, TimeoutError, IOError, aiohttp.ClientError) as ex:
        await session.close()
        raise ImageProxyError(str(ex))
    except Exception:
        await session.close()
        raise
    if yarl.URL(response.url) != yarl.URL(url):
        try:
            await check_private_address(str(response.url))
        except Exception:
            await session.close()
            raise
    return response 

def search(self,query,page):
        params = {
            "Query":query,
            "$skip": self.parameters["$top"] * page
        }
        params.update(self.parameters)
        try:
            r = yield from aiohttp.request(
                    'get',
                    self.url,
                    params=params,
                    headers=self.headers
                    )
            results = yield from r.json()
            yield from self.__process(results)
        except aiohttp.ClientError as client_error:
            print("Error: {emsg}".format(emsg=client_error)) 

def _fetch(self, resource: str,) -> Dict[Any, Any]:
        """ Fetch JSON data from a web or file resource and return a dict """
        logger.debug(f"fetching {resource}")
        if resource.startswith("http"):
            try:
                async with aiohttp.ClientSession() as session:
                    async with session.get(
                        resource, timeout=self.fetch_timeout
                    ) as resp:
                        if not resp.status == 200:
                            raise Exception(f"Fetch failed {resp.status}: {resource}")
                        data = await resp.json()
            except asyncio.TimeoutError:
                raise Exception(f"Request timed out to {resource}") from None
            except aiohttp.ClientError as exc:
                raise Exception(f"Client error {exc}, {resource}") from None
        else:
            with open(resource, "rt") as f:
                data = json.loads(f.read())

        return data 

def test_perform_request_ssl_error(auto_close, loop):
    for exc, expected in [
        (aiohttp.ClientConnectorCertificateError(mock.Mock(), mock.Mock()), SSLError),  # noqa
        (aiohttp.ClientConnectorSSLError(mock.Mock(), mock.Mock()), SSLError),
        (aiohttp.ClientSSLError(mock.Mock(), mock.Mock()), SSLError),
        (aiohttp.ClientError('Other'), ConnectionError),
        (asyncio.TimeoutError, ConnectionTimeout),
    ]:
        session = aiohttp.ClientSession(loop=loop)

        async def coro(*args, **Kwargs):
            raise exc

        session._request = coro

        conn = auto_close(AIOHttpConnection(session=session, loop=loop,
                                            use_ssl=True))
        with pytest.raises(expected):
            await conn.perform_request('HEAD', '/') 

def async_update(self):
    _LOGGER.info("Updating".format(self.query_url()))
    try:
      with async_timeout.timeout(TIMEOUT, loop=self.hass.loop):
        response = await self.websession.get(self.query_url())
        self.data = await response.json()
        _LOGGER.debug("Updating sensor: {}".format(self.data))
        _LOGGER.debug("next departure: {}".format(self.next_departure))
    except (asyncio.TimeoutError, aiohttp.ClientError, IndexError) as error:
      _LOGGER.error("Failed getting devices: %s", error) 

def fetch_json(self, url):
        """Request json from url."""
        data = None
        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(url) as resp:
                    data = await resp.json()
                    if resp.status != 200:
                        raise ServerError(data)
        except aiohttp.ClientError:
            raise ServerError(data)
        except json.JSONDecodeError:
            raise ServerError(data)
        return data 

def fetch_multi(self, urls):
        """Perform parallel fetch"""
        results = []
        error_msg = None
        try:
            async with aiohttp.ClientSession() as session:
                for url in urls:
                    await asyncio.sleep(0)
                    body = await self.fetch_with_session(session, url)
                    results.append(body)
        except asyncio.TimeoutError:
            error_msg = 'Request timed out'
            raise ClashRoyaleAPIError(message=error_msg)
        except aiohttp.ServerDisconnectedError as err:
            error_msg = 'Server disconnected error: {}'.format(err)
            raise ClashRoyaleAPIError(message=error_msg)
        except (aiohttp.ClientError, ValueError) as err:
            error_msg = 'Request connection error: {}'.format(err)
            raise ClashRoyaleAPIError(message=error_msg)
        except json.JSONDecodeError:
            error_msg = "Non JSON returned"
            raise ClashRoyaleAPIError(message=error_msg)
        else:
            return results
        finally:
            if error_msg is not None:
                raise ClashRoyaleAPIError(message=error_msg) 

def _do_http_request(self, url, body, headers):
        try:
            return await self._do_http_request_impl(url, body, headers)
        except asyncio.TimeoutError:
            raise PyVLXException("Request timeout when talking to VELUX API")
        except aiohttp.ClientError:
            raise PyVLXException("HTTP error when talking to VELUX API")
        except OSError:
            raise PyVLXException("OS error when talking to VELUX API") 

def get_guild(name, title_case=True, *, tries=5) -> Optional[Guild]:
    """Fetches a guild from TibiaData, parses and returns a Guild object

    The Guild object contains all the information available on Tibia.com
    Guilds are case sensitive on tibia.com so guildstats.eu is checked for correct case.
    If the guild can't be fetched due to a network error, an NetworkError exception is raised
    If the character doesn't exist, None is returned."""
    if tries == 0:
        raise errors.NetworkError(f"get_guild({name})")

    # Fix casing using guildstats.eu if needed
    # Sorry guildstats.eu :D
    try:
        guild = CACHE_GUILDS[name.lower()]
        return guild
    except KeyError:
        pass

    if not title_case:
        guild_name = await get_guild_name_from_guildstats(name, tries=tries)
        name = guild_name if guild_name else name
    else:
        name = name.title()

    # Fetch website
    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(Guild.get_url_tibiadata(name)) as resp:
                content = await resp.text(encoding='ISO-8859-1')
                guild = Guild.from_tibiadata(content)
    except (aiohttp.ClientError, asyncio.TimeoutError, tibiapy.TibiapyException):
        await asyncio.sleep(config.network_retry_delay)
        return await get_guild(name, title_case, tries=tries - 1)

    if guild is None:
        if title_case:
            return await get_guild(name, False)
        else:
            return None
    CACHE_GUILDS[name.lower()] = guild
    return guild 

def get_access_token(self) -> None:
        """
        Get a Reddit API OAuth2 access token and assign it to self.access_token.

        A token is valid for 1 hour. There will be MAX_RETRIES to get a token, after which the cog
        will be unloaded and a ClientError raised if retrieval was still unsuccessful.
        """
        for i in range(1, self.MAX_RETRIES + 1):
            response = await self.bot.http_session.post(
                url=f"{self.URL}/api/v1/access_token",
                headers=self.HEADERS,
                auth=self.client_auth,
                data={
                    "grant_type": "client_credentials",
                    "duration": "temporary"
                }
            )

            if response.status == 200 and response.content_type == "application/json":
                content = await response.json()
                expiration = int(content["expires_in"]) - 60  # Subtract 1 minute for leeway.
                self.access_token = AccessToken(
                    token=content["access_token"],
                    expires_at=datetime.utcnow() + timedelta(seconds=expiration)
                )

                log.debug(f"New token acquired; expires on UTC {self.access_token.expires_at}")
                return
            else:
                log.debug(
                    f"Failed to get an access token: "
                    f"status {response.status} & content type {response.content_type}; "
                    f"retrying ({i}/{self.MAX_RETRIES})"
                )

            await asyncio.sleep(3)

        self.bot.remove_cog(self.qualified_name)
        raise ClientError("Authentication with the Reddit API failed. Unloading the cog.") 

def fetch(self, session, block_nb):
        try:
            async with session.post(self.__class__.eth_url,
                                    data=Ethdrain.make_request(block_nb),
                                    headers={"content-type": "application/json"}) as response:
                for data_store in self.data_stores:
                    data_store.extract(await response.json())

        except (aiohttp.ClientError, asyncio.TimeoutError) as exception:
            logging.error("block: " + str(block_nb))
            print("Issue with block {}:\n{}\n".format(block_nb, exception)) 

def _http_request(
        trace_config,
        url: str,
        method: str = "GET",
        status_code: int = HTTPStatus.OK,
        request_handler: typing.Callable = None,
        **kwargs
    ) -> typing.Tuple[str, int]:
        """Helper to start an aiohttp test server and send an actual HTTP request to it."""

        async def do_request():
            async def default_handler(request):
                assert "traceparent" in request.headers
                return aiohttp.web.Response(status=int(status_code))

            handler = request_handler or default_handler

            app = aiohttp.web.Application()
            parsed_url = urllib.parse.urlparse(url)
            app.add_routes([aiohttp.web.get(parsed_url.path, handler)])
            app.add_routes([aiohttp.web.post(parsed_url.path, handler)])
            app.add_routes([aiohttp.web.patch(parsed_url.path, handler)])

            with contextlib.suppress(aiohttp.ClientError):
                async with aiohttp.test_utils.TestServer(app) as server:
                    netloc = (server.host, server.port)
                    async with aiohttp.test_utils.TestClient(
                        server, trace_configs=[trace_config]
                    ) as client:
                        await client.start_server()
                        await client.request(
                            method, url, trace_request_ctx={}, **kwargs
                        )
            return netloc

        loop = asyncio.get_event_loop()
        return loop.run_until_complete(do_request()) 

def __init__(
        self,
        seconds: Union[float, int],
        hours: Union[float, int],
        minutes: Union[float, int],
        coro: Coroutine,
        count: Optional[int],
        reconnect: bool,
        loop: asyncio.AbstractEventLoop,
    ) -> None:
        self.coro = coro
        self.reconnect = reconnect
        self.loop = loop or gd.utils.acquire_loop()
        self.count = count
        self._current_loop = 0
        self._task = None
        self._injected = None
        self._valid_exception = (
            OSError,
            gd.ClientException,
            aiohttp.ClientError,
            asyncio.TimeoutError,
        )

        self._before_loop = None
        self._after_loop = None
        self._is_being_cancelled = False
        self._has_failed = False
        self._stop_next_iteration = False

        if self.count is not None and self.count <= 0:
            raise ValueError("Count must be greater than 0 or None.")

        self.change_interval(seconds=seconds, minutes=minutes, hours=hours)

        if not inspect.iscoroutinefunction(self.coro):
            raise TypeError(f"Expected coroutine function, not {type(self.coro).__name__!r}.") 

def _send_request(self, post_data):
        """
        Send RPC POST request to daemon

        post_data: Any valid RPC request as JSON string

        If applicable, returns response['arguments']['torrents'] or
        response['arguments'], otherwise response.

        Raises ClientError.
        """
        import aiohttp
        try:
            answer = await self._post(post_data)

        # NOTE #163: Letting asyncio.CancelledError bubble up seems to fix the issue that
        #            causes empty torrent lists in new tabs until the next poll iteration.
        #            But I've seen this error pop up in the TUI log: "Unclosed client
        #            session client_session: <aiohttp.client.ClientSession object at
        #            0x7f35d98d1be0>" This may or may not be related.
        except aiohttp.ClientError as e:
            log.debug('Caught during POST request: %r', e)
            raise ConnectionError(self.url)

        except asyncio.TimeoutError:
            raise TimeoutError(self.timeout, self.url)

        else:
            if answer['result'] != 'success':
                raise RPCError(answer['result'].capitalize())
            else:
                if 'arguments' in answer:
                    if 'torrents' in answer['arguments']:
                        return answer['arguments']['torrents']
                    else:
                        return answer['arguments']
                return answer 

def check_for_stale_cache(
        self, path: str, required_serial: Optional[int], got_serial: Optional[int]
    ) -> None:
        # The PYPI-LAST-SERIAL header allows us to identify cached entries,
        # e.g. via the public CDN or private, transparent mirrors and avoid us
        # injecting stale entries into the mirror without noticing.
        if required_serial is not None:
            # I am not making required_serial an optional argument because I
            # want you to think really hard before passing in None. This is a
            # really important check to achieve consistency and you should only
            # leave it out if you know what you're doing.
            if not got_serial or got_serial < required_serial:
                logger.debug(
                    f"Expected PyPI serial {required_serial} for request {path} "
                    + f"but got {got_serial}"
                )

                # HACK: The following attempts to purge the cache of the page we
                # just tried to fetch. This works around PyPI's caches sometimes
                # returning a stale serial for a package. Ideally, this should
                # be fixed on the PyPI side, at which point the following code
                # should be removed.
                # Timeout: uses self.sessions's timeout value
                logger.debug(f"Issuing a PURGE for {path} to clear the cache")
                try:
                    async with self.session.request("PURGE", path):
                        pass
                except (aiohttp.ClientError, asyncio.TimeoutError):
                    logger.warning(
                        "Got an error when attempting to clear the cache", exc_info=True
                    )

                raise StalePage(
                    f"Expected PyPI serial {required_serial} for request {path} "
                    + f"but got {got_serial}. "
                    + "HTTP PURGE has been issued to the request url"
                ) 

def test_request_fails_client_error() -> None:
    """Test a successful call of request."""
    session = CoroutineMock(side_effect=aiohttp.ClientError)

    with pytest.raises(errors.RequestError) as e_info:
        await utils.async_request(session, "url")

    assert str(e_info.value) == "Error requesting data from url: " 

def produce_blocks(self):
        for nr, block in enumerate(self.range.subranges(self.block_size)):
            with tqdm(
                    desc=f'Block #{nr}',
                    leave=False,
                    dynamic_ncols=True,
                    total=block.size(),
                    unit='B',
                    unit_scale=True,
                    unit_divisor=1024) as bar, hook_print(bar.write):

                self.buffer = BytesIO()
                futures = [
                    asyncio.ensure_future(
                        self.request_piece(r, block.begin, bar))
                    for r in block.subranges(self.piece_size)
                ]
                try:
                    await asyncio.gather(*futures)
                    await self.blocks.put(self.buffer.getvalue())
                except (asyncio.CancelledError, aiohttp.ClientError) as exc:
                    for f in futures:
                        f.cancel()
                    #  Notify the consumer to leave
                    #  -- which is waiting at the end of this queue!
                    await self.blocks.put(exc)
                    return 

def handle_exception():
    """
    Context manager translating network related exceptions
    to custom :mod:`~galaxy.api.errors`.
    """
    try:
        yield
    except asyncio.TimeoutError:
        raise BackendTimeout()
    except aiohttp.ServerDisconnectedError:
        raise BackendNotAvailable()
    except aiohttp.ClientConnectionError:
        raise NetworkError()
    except aiohttp.ContentTypeError:
        raise UnknownBackendResponse()
    except aiohttp.ClientResponseError as error:
        if error.status == HTTPStatus.UNAUTHORIZED:
            raise AuthenticationRequired()
        if error.status == HTTPStatus.FORBIDDEN:
            raise AccessDenied()
        if error.status == HTTPStatus.SERVICE_UNAVAILABLE:
            raise BackendNotAvailable()
        if error.status == HTTPStatus.TOO_MANY_REQUESTS:
            raise TooManyRequests()
        if error.status >= 500:
            raise BackendError()
        if error.status >= 400:
            logging.warning(
                "Got status %d while performing %s request for %s",
                error.status, error.request_info.method, str(error.request_info.url)
            )
            raise UnknownError()
    except aiohttp.ClientError:
        logging.exception("Caught exception while performing request")
        raise UnknownError()