Python requests.response() Examples

def response_data(response, bad_json=False):
        """
        Extracts data from a requests response object
        :param response: The response from which we need to extract information
        :type response: requests.response
        :param bad_json: Default False: If true, uses ast eval instead of json
        load
        :type bad_json bool
        :return: response result in Pythonic form, if it can get it, Else None
        :raises Exception
        """
        data = None
        if response:
            if not bad_json:
                try:
                    data = json_to_python(response.text)
                except Exception:
                    data = parse_literals(response.text)
            else:
                data = parse_literals(response.text)
        return data 

def test_login_shorten(self):
        response = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        with requests.Session() as sess:
            response = sess.post(self.url + '/login', data=self.login_data, headers=self.headers)
            self.cookies = sess.cookies
            self.assertTrue('Welcome Test' in response.text)

            # Shorten the URL
            data = self.data
            data['long_url'] = 'https://example.com/1'
            response = sess.post(self.url + '/shorten', data=data, headers=self.headers)
            short_url = self._get_short_url_from_response(response)

            self.assertEqual(response.status_code, 200)
            self.assertTrue('value="{}"'.format(short_url) in response.text)
            self.assertTrue('Welcome Test' in response.text)
            self.assertTrue('Copy To Clipboard' in response.text)
            self.assertTrue('Shorten Another Link' in response.text)

            # verify its on dashboard
            response = sess.get(self.url + '/dashboard')
            short_code = short_url.split('/')[-1]
            self.assertEqual(response.status_code, 200)
            self.assertTrue('{}+'.format(short_code) in response.text)
            self.assertTrue('https://example.com/1' in response.text) 

def test_link_hits(self):
        data = self.data
        data['long_url'] = 'http://example.com/index'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        short_url = self._get_short_url_from_response(response)

        # Open link
        for i in range(2):
            requests.get(short_url)
            stats_page = requests.get(short_url + '+')
            self.assertEqual(stats_page.status_code, 200)
            self.assertTrue('Total Hits: {}'.format(i+1) in stats_page.text)

    # def test_link_stats(self):
    #     pass
    #
    # def test_secret_link_stats(self):
    #     pass
    #
    # def test_expired_link_stats(self):
    #     pass
    # 

def test_custom_link_stats(self):
        data = self.data
        data['custom_url'] = 'ninja'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        short_url = self._get_short_url_from_response(response)

        # Open link
        for i in range(2):
            requests.get(short_url)
            stats_page = requests.get(short_url + '+')
            self.assertEqual(stats_page.status_code, 200)
            self.assertTrue('Total Hits: {}'.format(i+1) in stats_page.text)

    # #######################
    # # Test static resources
    # ####################### 

def test_shorten(self):
        data = self.data
        response = self.client.call('/shorten', data=data, headers=self.headers)
        short_url = self._get_short_url_from_response(response)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('value="{}"'.format(short_url) in response.text)
        self.assertTrue('Copy To Clipboard' in response.text)
        self.assertTrue('Shorten Another Link' in response.text)
        self.assertTrue('{}+'.format(short_url) in response.text)

        # Repeat shorten should return the same result
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('value="{}"'.format(short_url) in response.text)

        # Test a different url
        data['long_url'] = 'https://www.python.org/'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        short_url = self._get_short_url_from_response(response)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('value="{}"'.format(short_url) in response.text) 

def update_resource_version(self, owner_type, owner_id,
                                resource_id, version_id, resource_type, base_data):
        """
        Update source version. Limits update to only the description and released fields for now.
        :param owner_type: 'orgs' or 'users'
        :param owner_id: ID of the org/user owner
        :param resource_id: ID of the source/collection
        :param version_id: ID of the source/collection_version
        :param resource_type: 'source' or 'collection'
        :param base_data: Dictionary of fields to update
        :returns: response object
        """
        data = {}
        if 'description' in base_data:
            data['description'] = base_data['description']
        if 'released' in base_data:
            data['released'] = base_data['released']
        if 'retired' in base_data:
            data['retired'] = base_data['retired']
        if 'version_external_id' in base_data:
            data['version_external_id'] = base_data['version_external_id']

        result = self.put(owner_type, owner_id, resource_type, resource_id, version_id, **data)
        return result 

def get_json(self, *args):
        """
        Smarter GET request when you really want a json object back.
        Note: This is experimental -- not sure if this is the right abstraction.
        :param *args: All positional arguments are appended to the request URL.
        :returns: json string or None if error.
        :exception: Will raise exception if response status code is not 200.
        """
        results = self.get(*args)
        if results.status_code != requests.codes.ok:
            results.raise_for_status()
        if len(results.content) > 0:
            return results.json()
        else:
            return None


    # TODO: Retire get_by_url? 

def head(self, *args, **kwargs):
        """
        Issue HEAD request to API.
        :param *args: All positional arguments are appended to the request URL.
        :param **kwargs: These are not used at the moment, since this is a get request TODO
        :returns: requests.response object.
        """
        self.url = '%s/' % (self.host)
        if len(args) > 0:
            self.url = self.url + '/'.join(args) + '/'
        if self.debug:
            self.logger.debug('HEAD %s %s %s' % (self.url, json.dumps(kwargs), self.headers))

        # look for optional keyword argument params for constructing URL param
        # i.e. ?f1=v1&f2=v2
        params = kwargs.get('params')

        results = requests.head(self.url, params=params,
                                headers=self.headers)
        self.status_code = results.status_code
        if self.debug:
            self.debug_result(results)
        return results 

def post(self, type_name, *args, **kwargs):
        """
        Issue POST request to API.
        :param type_name: is a string specifying the type of the object according to the API.
        :param *args: The rest of the positional arguments will be appended to the post URL
        :param *kwargs: all the keyword arguments will become post data.
        :returns: response object from requests.
        """
        url = '%s/%s/' % (self.host, type_name)
        if len(args) > 0:
            url = url + '/'.join(args) + '/'
        if self.debug:
            self.logger.debug('POST %s %s %s' % (url, json.dumps(kwargs),
                                                 self.headers))

        results = requests.post(url, data=json.dumps(kwargs),
                                headers=self.headers)
        self.status_code = results.status_code
        if self.debug:
            self.debug_result(results)
        return results 

def requests_link(url: str, encoding: str = "utf-8", method: str = "get", data: Dict = None, headers: Dict = None):
    """
    利用 requests 请求网站, 爬取网站内容, 如网站链接失败, 可重复爬取 20 次
    :param url: string 网站地址
    :param encoding: string 编码类型: "utf-8", "gbk", "gb2312"
    :param method: string 访问方法: "get", "post"
    :param data: dict 上传数据: 键值对
    :param headers: dict 游览器请求头: 键值对
    :return: requests.response 爬取返回内容: response
    """
    i = 0
    while True:
        try:
            if method == "get":
                r = requests.get(url, timeout=20)
                r.encoding = encoding
                return r
            elif method == "post":
                r = requests.post(url, timeout=20, data=data, headers=headers)
                r.encoding = encoding
                return r
            else:
                raise ValueError("请提供正确的请求方式")
        except requests.exceptions.Timeout as e:
            i += 1
            print(f"第{str(i)}次链接失败, 最多尝试 20 次", e)
            time.sleep(5)
            if i > 20:
                return None 

def fetch(self, fmt_p=None, fmt_j=None):
        """Fetches latest data

        Prepares the query URL based on self.params and executes the request

        Returns
        -------
        response: requests.response
            A response object
        """
        url = self._prepare_query()
        data = self._execute_iex_query(url)
        return self._output_format(data, fmt_j=fmt_j, fmt_p=fmt_p) 

def _handle_error(self, response):
        """
        Handles all responses which return an error status code
        """
        raise IEXQueryError(response.status_code, response.text) 

def _execute_iex_query(self, url):
        """ Executes HTTP Request
        Given a URL, execute HTTP request from IEX server. If request is
        unsuccessful, attempt is made self.retry_count times with pause of
        self.pause in between.

        Parameters
        ----------
        url: str
            A properly-formatted url

        Returns
        -------
        response: requests.response
            Sends requests.response object to validator

        Raises
        ------
        IEXQueryError
            If problems arise when making the query
        """
        params = self.params
        params["token"] = self.token
        for _ in range(self.retry_count + 1):
            response = self.session.get(url=url, params=params)
            logger.debug("REQUEST: %s" % response.request.url)
            logger.debug("RESPONSE: %s" % response.status_code)
            if response.status_code == requests.codes.ok:
                return self._validate_response(response)
            time.sleep(self.pause)
        return self._handle_error(response) 

def _validate_response(self, response):
        """ Ensures response from IEX server is valid.

        Parameters
        ----------
        response: requests.response
            A requests.response object

        Returns
        -------
        response: Parsed JSON
            A json-formatted response

        Raises
        ------
        ValueError
            If a single Share symbol is invalid
        IEXQueryError
            If the JSON response is empty or throws an error

        """
        # log the number of messages used
        key = "iexcloud-messages-used"
        if key in response.headers:
            msg = response.headers[key]
        else:
            msg = "N/A"
        logger.info("MESSAGES USED: %s" % msg)

        if response.text == "Unknown symbol":
            raise IEXQueryError(response.status_code, response.text)
        try:
            json_response = response.json(
                parse_int=self.json_parse_int, parse_float=self.json_parse_float
            )
            if isinstance(json_response, str) and ("Error Message" in json_response):
                raise IEXQueryError(response.status_code, response.text)
        except ValueError:
            raise IEXQueryError(response.status_code, response.text)
        return json_response 

def test_logo_svg(self):
        response = requests.get(self.url + '/static/logo/logov2.svg')
        self.assertEqual(response.status_code, 200) 

def pandas_read_html_link(url: str, encoding: str = "utf-8", method: str = "get", data: Dict = None, headers: Dict = None):
    """
    利用 pandas 提供的 read_html 函数来直接提取网页中的表格内容, 如网站链接失败, 可重复爬取 20 次
    :param url: string 网站地址
    :param encoding: string 编码类型: "utf-8", "gbk", "gb2312"
    :param method: string 访问方法: "get", "post"
    :param data: dict 上传数据: 键值对
    :param headers: dict 游览器请求头: 键值对
    :return: requests.response 爬取返回内容: response
    """
    i = 0
    while True:
        try:
            if method == "get":
                r = requests.get(url, timeout=20)
                r.encoding = encoding
                r = pd.read_html(r.text, encoding=encoding)
                return r
            elif method == "post":
                r = requests.post(url, timeout=20, data=data, headers=headers)
                r.encoding = encoding
                r = pd.read_html(r.text, encoding=encoding)
                return r
            else:
                raise ValueError("请提供正确的请求方式")
        except requests.exceptions.Timeout as e:
            i += 1
            print(f"第{str(i)}次链接失败, 最多尝试20次", e)
            time.sleep(5)
            if i > 20:
                return None 

def test_custom_taken_link_shorten(self):
        custom_code = 'go'
        response = requests.get(self.url + '/check?custom_code={}'.format(custom_code))
        self.assertTrue(response.json().get('ok'))

        data = self.data
        data['custom_url'] = custom_code
        requests.post(self.url + '/shorten', data=data, headers=self.headers)

        response = requests.get(self.url + '/check?custom_code={}'.format(custom_code))
        self.assertFalse(response.json().get('ok'))

        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 400) 

def test_custom_taken_link_availability(self):
        custom_code = 'logo'
        response = requests.get(self.url + '/check?custom_code={}'.format(custom_code))
        self.assertTrue(response.json().get('ok'))
        data = self.data
        data['custom_url'] = custom_code
        requests.post(self.url + '/shorten', data=data, headers=self.headers)
        response = requests.get(self.url + '/check?custom_code={}'.format(custom_code))
        self.assertFalse(response.json().get('ok')) 

def test_non_loggedin_dashboard(self):
        response = requests.get(self.url + '/dashboard')
        self.assertTrue(response.status_code == 400)
        self.assertTrue('Please log in again to continue.</h3>' in response.text)
    #
    # ###############
    # # Link Options
    # ############### 

def test_invalid_password_login(self):
        response = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        login_data = self.login_data.copy()
        login_data['password'] = 'i know it'
        response = requests.post(self.url + '/login', data=login_data, headers=self.headers)
        self.assertEqual(response.status_code, 400)
        self.assertTrue('LOGIN' in response.text)
        self.assertTrue('Invalid username or password.' in response.text) 

def test_logout(self):
        _ = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        with requests.Session() as sess:
            _ = sess.post(self.url + '/login', data=self.login_data, headers=self.headers)
            self.cookies = sess.cookies
            response = sess.get(self.url)
            self.assertEqual(response.status_code, 200)
            self.assertTrue('Welcome Test' in response.text)
            # logout
            response = sess.get(self.url + '/logout', headers=self.headers)
            self.assertEqual(response.url.strip('/'), self.url)
            self.assertEqual(response.status_code, 200)
            self.assertTrue('Welcome Test' not in response.text) 

def test_login(self):
        response = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        response = requests.get(self.url)
        self.assertTrue('LOGIN' in response.text)
        self.assertTrue('DASHBOARD' not in response.text)

        """Test redirection, hidden login, visible dashboard section, Welcome <username> section
        and dashboard table"""
        response = requests.post(self.url + '/login', data=self.login_data, headers=self.headers)
        self.assertEqual(response.status_code, 200)
        self.assertTrue(response.url == self.url + '/dashboard')
        self.assertTrue('DASHBOARD' in response.text)
        self.assertTrue('LOGIN' not in response.text)
        self.assertTrue('Welcome Test' in response.text)
        self.assertTrue('<table class="table table-bordered">' in response.text) 

def check_response(self, response):
        """
        Checks the status code and raise an AirflowException exception on non 2XX or 3XX
        status codes

        :param response: A requests response object
        :type response: requests.response
        """
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            self.log.error("HTTP error: %s", response.reason)
            self.log.error(response.text)
            raise AirflowException(str(response.status_code) + ":" + response.reason) 

def test_signup(self):
        data = self.user_data
        data['email'] = 'ninja@example.com'
        response = requests.post(self.url + '/signup', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('Welcome Test' in response.text)
        self.assertIsNotNone(response.cookies.get('access_token'))

        response = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        self.assertEqual(response.status_code, 400)
        self.assertTrue('User exists' in response.text) 

def test_unshorten(self):
        data = self.data
        data['long_url'] = 'https://github.com'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 200)
        short_url = self._get_short_url_from_response(response)

        response = requests.get(short_url)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.url, data['long_url'])

    # User access 

def test_already_shortened_url_error(self):
        err_msg = 'URL is already a pygmy shortened link'
        data = self.data
        data['long_url'] = 'https://pygy.co'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 400)
        self.assertTrue(err_msg in response.text) 

def _get_short_url_from_response(self, response):
        """
        :param response: requests.response object
        :return: str
        """
        # TODO: Fix this
        resp_text = response.text
        idx = resp_text.find(self.url)
        idx_end = resp_text.find('" readonly autofocus id="short_url_blocked"')
        return resp_text[idx:idx_end] 

def create_source_version(self, owner_type, org_id, source_id, base_data):
        """
        Create a new source version.
        :param owner_type: 'orgs' or 'users'
        :param owner_id: ID of the org/user/ owner
        :param source_id: ID of the source
        :param base_data: Dictionary of fields for the new source version
        :returns: response object
        """
        data = {}
        data.update(base_data)
        result = self.post(owner_type, org_id, 'sources', source_id, 'versions', **data)
        return result 

def update_source(self, owner_type, owner_id, source_id, base_data, extras=[]):
        """
        Update source owned by org.
        :param owner_type: 'orgs' or 'users'
        :param owner_id: ID of the org/user/ owner
        :param base_data: is a dictionary of fields.
        :param extras: Extras to save to the resource
        :returns: response object.
        """
        data = {}
        data.update(base_data)
        result = self.put(owner_type, owner_id, 'sources', source_id, **data)
        return result 

def create_source(self, owner_type, owner_id, base_data, extras=[]):
        """
        Create source.
        :param owner_type: 'orgs' or 'users'
        :param owner_id: ID of the org/user/ owner
        :param base_data: Dictionary of fields for the new source version
        :param extras: Extras to save to the resource
        :returns: response object

        TODO(paynejd): create_sources extras not implemented
        """
        data = {}
        data.update(base_data)
        result = self.post(owner_type, owner_id, 'sources', **data)
        return result