Python httpx.get() Examples

def _resolve_github_ref(owner: str, repo: str, ref: str) -> str:
    """
    Given a GitHub owner/repo/ref, return the sha1 at that ref.

    Raise WorkbenchModuleImportError if the HTTP request fails or GitHub says
    there is no such ref. These errors are all in English, since we assume most
    module authors can read English and it takes effort to translate technical
    messages.
    """
    try:
        # raise HTTPError
        response = httpx.get(
            "https://api.github.com/repos/%s/%s/git/ref/heads/%s" % (owner, repo, ref),
            headers=[("Accept", "application/vnd.github.v3+json")],
        )
        # raise HTTPError
        response.raise_for_status()
    except httpx.HTTPError as err:
        raise WorkbenchModuleImportError(
            "HTTP error asking GitHub to resolve ref %(ref)s: %(message)s"
            % {"ref": ref, "message": str(err)}
        )

    data = json.loads(response.text)
    return data["object"]["sha"] 

def fetch(
        self,
        station: str = None,
        lat: float = None,
        lon: float = None,
        timeout: int = 10,
    ) -> str:
        """
        Fetches a report string from the service
        """
        if station:
            valid_station(station)
        elif lat is None or lon is None:
            raise ValueError("No valid fetch parameters")
        try:
            url, params = self._make_url(station, lat, lon)
            if self.method.lower() == "post":
                resp = httpx.post(
                    url, params=params, data=self._post_data(station), timeout=timeout
                )
            else:
                resp = httpx.get(url, params=params, timeout=timeout)
            if resp.status_code != 200:
                raise SourceError(
                    f"{self.__class__.__name__} server returned {resp.status_code}"
                )
        except (httpx.ConnectTimeout, httpx.ReadTimeout):
            raise TimeoutError(f"Timeout from {self.__class__.__name__} server")
        except gaierror:
            raise ConnectionError(
                f"Unable to connect to {self.__class__.__name__} server"
            )
        report = self._extract(resp.text, station)
        return self._clean_report(report) 

def async_fetch(
        self,
        station: str = None,
        lat: float = None,
        lon: float = None,
        timeout: int = 10,
    ) -> str:
        """
        Asynchronously fetch a report string from the service
        """
        if station:
            valid_station(station)
        elif lat is None or lon is None:
            raise ValueError("No valid fetch parameters")
        url, params = self._make_url(station, lat, lon)
        try:
            async with httpx.AsyncClient(timeout=timeout) as client:
                if self.method.lower() == "post":
                    resp = await client.post(
                        url, params=params, data=self._post_data(station)
                    )
                else:
                    resp = await client.get(url, params=params)
                if resp.status_code != 200:
                    raise SourceError(
                        f"{self.__class__.__name__} server returned {resp.status_code}"
                    )
        except (httpx.ConnectTimeout, httpx.ReadTimeout):
            raise TimeoutError(f"Timeout from {self.__class__.__name__} server")
        except gaierror:
            raise ConnectionError(
                f"Unable to connect to {self.__class__.__name__} server"
            )
        report = self._extract(resp.text, station)
        return self._clean_report(report)


# Multiple sources for NOAA data 

def get_service(station: str, country_code: str) -> Service:
    """
    Returns the preferred service for a given station
    """
    for prefix in PREFERRED:
        if station.startswith(prefix):
            return PREFERRED[prefix]
    return BY_COUNTRY.get(country_code, NOAA)


# Specialty Services 

def get_content_from_url(self, url):
        """
        Fetch url and try to get the title and description from the response
        """
        title = None
        description = None
        timeout = httpx.Timeout(10.0, connect_timeout=2.0, read_timeout=5.0)
        try:
            r = httpx.get(url, timeout=timeout)
        except Exception as e:
            self.logger.error(f"Failed fetching url {url}. Error: {e}")
            return (title, description)

        if r.status_code != 200:
            self.logger.info(f"Failed fetching url {url}. Status code: {r.status_code}")
            return (title, description)

        # try parse and get the title
        try:
            soup = BeautifulSoup(r.text, "html.parser")
            if soup.head and soup.head.title:
                title = soup.head.title.string.strip()
            descr_tag = soup.find("meta", attrs={"name": "description"})
            if descr_tag:
                description = descr_tag.get("content", None)
        except Exception as e:
            self.logger.error(f"Failed parsing response from url {url}. Error: {e}")
            return (title, description)

        # Issue 63 patch - Title should not contain newlines or tabs
        if title is not None:
            assert isinstance(title, str)
            title = title.replace('\n', '')
            title = title.replace('\t', '')
        return (title, description) 

def user_profile(access_token: str, domain: str) -> Dict[str, Any]:
    """Returns user profile."""

    headers = {
        "Authorization": f"Bearer {access_token}"
    }

    resp = httpx.get(
        f"https://api.amazon.{domain}/user/profile", headers=headers
    )
    resp.raise_for_status()

    return resp.json() 

def default_captcha_callback(captcha_url: str) -> str:
    """Helper function for handling captcha."""
    # on error print captcha url instead of display captcha
    try:
        captcha = httpx.get(captcha_url).content
        f = io.BytesIO(captcha)
        img = Image.open(f)
        img.show()
    except:
        print(captcha_url)

    guess = input("Answer for CAPTCHA: ")
    return str(guess).strip().lower() 

def pr_status():
    """Show status of PR found in requirement files."""
    for pr in search_prs():
        r = httpx.get(f"https://api.github.com/repos/{pr.org}/{pr.repo}/pulls/{pr.pr}")
        r.raise_for_status()
        state = display_state(r.json())
        click.echo(f"https://github.com/{pr.org}/{pr.repo}/pull/{pr.pr} is {state}") 

def set_settings(self, data):
        super().set_settings(data)
        if data.get("status"):
            self.status = data["status"]
        if data.get("type"):
            self.type = data["type"] 

def help(self):
        return "If I see a url in a message I will try to get the title from the page and spit it out" 

def autodetect_locale(domain: str) -> Dict[str, str]:
    """
    Try to automatically detect correct settings for marketplace.

    Needs the top level domain of the audible page to continue with
    (e.g. co.uk, co.jp) and returns results found.

    """
    domain = domain.lstrip(".")
    site = f"https://www.audible.{domain}"
    params = {
        "ipRedirectOverride": True,
        "overrideBaseCountry": True
    }

    try:
        resp = httpx.get(site, params=params)
    except ConnectError as e:
        logger.warning(f"site {site} doesn\'t exists or Network Error occours")
        raise e

    soup = BeautifulSoup(resp.text, "html.parser")

    login_link = soup.find("a", class_="ui-it-sign-in-link")["href"]
    parsed_link = urlparse(login_link)
    query_string = parse_qs(parsed_link.query)
    market_place_id = query_string['marketPlaceId'][0]
    country_code = query_string['pageId'][0].split("_")[-1]

    return {
        "country_code": country_code,
        "domain": domain,
        "market_place_id": market_place_id
    } 

def __init__(self, request_type: str):
        super().__init__(self._rtype_map.get(request_type, request_type)) 

def text_cb(self, room, event):
        """
        Handle client callbacks for all room text events
        """
        if self.bot.should_ignore_event(event):
            return

        # no content at all?
        if len(event.body) < 1:
            return

        # are we on in this room?
        status = self.status.get(room.room_id, "OFF")
        if status not in self.STATUSES:
            return
        if status == "OFF":
            return

        # extract possible urls from message
        urls = re.findall(r"(https?://\S+)", event.body)

        # no urls, nothing to do
        if len(urls) == 0:
            return

        # fetch the urls and if we can see a title spit it out
        for url in urls:
            try:
                title, description = self.get_content_from_url(url)
            except Exception:
                # failed fetching, give up
                continue

            msg = None

            if status == "TITLE" and title is not None:
                msg = f"Title: {title}"
            elif status == "DESCRIPTION" and description is not None:
                msg = f"Description: {description}"

            elif status == "BOTH" and title is not None and description is not None:
                msg = f"Title: {title}\nDescription: {description}"

            elif status == "BOTH" and title is not None:
                msg = f"Title: {title}"
            elif status == "BOTH" and description is not None:
                msg = f"Description: {description}"

            if msg is not None:
                await self.bot.send_text(room, msg, msgtype=self.type, bot_ignore=True) 

def matrix_message(self, bot, room, event):
        """
        commands for setting what to do in this channel
        """
        bot.must_be_admin(room, event)

        args = shlex.split(event.body)
        args.pop(0)

        # save the new status
        if len(args) == 1 and self.STATUSES.get(args[0].upper()) is not None:
            self.status[room.room_id] = args[0].upper()
            bot.save_settings()
            await bot.send_text(
                room, f"Ok, {self.STATUSES.get(self.status[room.room_id])}"
            )
            return

        # show status
        elif len(args) == 1 and args[0] == "status":
            await bot.send_text(
                room, self.STATUSES.get(self.status.get(room.room_id, "OFF"))
            )
            return

        # set type to notice
        elif len(args) == 1 and args[0] == "notice":
            bot.must_be_owner(event)
            self.type = "m.notice"
            bot.save_settings()
            await bot.send_text(
                room, "Sending titles as notices from now on."
            )
            return

        # show status
        elif len(args) == 1 and args[0] == "text":
            bot.must_be_owner(event)
            self.type = "m.text"
            bot.save_settings()
            await bot.send_text(
                room, "Sending titles as text from now on."
            )
            return

        # invalid command
        await bot.send_text(
            room,
            "Sorry, I did not understand. See README for command list.",
        )

        return