Python requests.exceptions.TooManyRedirects() Examples

def test_neutron_exception_is_raised_on_any_request_error(self):
        # timeout exception raises InfobloxTimeoutError
        f = mock.Mock()
        f.__name__ = 'mock'
        f.side_effect = req_exc.Timeout
        self.assertRaises(exceptions.InfobloxTimeoutError,
                          connector.reraise_neutron_exception(f))

        # all other request exception raises InfobloxConnectionError
        supported_exceptions = [req_exc.HTTPError,
                                req_exc.ConnectionError,
                                req_exc.ProxyError,
                                req_exc.SSLError,
                                req_exc.TooManyRedirects,
                                req_exc.InvalidURL]

        for ex in supported_exceptions:
            f.side_effect = ex
            self.assertRaises(exceptions.InfobloxConnectionError,
                              connector.reraise_neutron_exception(f)) 

def get_listings(start=1,limit=1000):
    parameters = {
        'start': start,
        'limit': str(limit),
        'convert': 'USD',
        'sort': 'market_cap'
    }
    #sort=market_cap&start=1&limit=10&cryptocurrency_type=tokens&convert=USD,BTC
    headers = {
        'Accepts': 'application/json',
        'X-CMC_PRO_API_KEY': cmc_key,
    }

    session = Session()
    session.headers.update(headers)

    endpoint_summary = 'cryptocurrency/listings/latest'
    try:
        url = base_url + endpoint_summary
        response = session.get(url, params=parameters)
        data = json.loads(response.text)["data"]
        return data
    except (ConnectionError, Timeout, TooManyRedirects) as e:
        print(e) 

def get_info(idlist):
    endpoint_description = "cryptocurrency/info"
    
    #[str(x)+',' for x in range(1,20)]
    parameters = {
        #'symbol': 'BTC,ETH,XRP,LTC'
        'id':idlist
    }
    headers = {
        'Accepts': 'application/json',
        'X-CMC_PRO_API_KEY': cmc_key,
    }

    session = Session()
    session.headers.update(headers)

    try:
        url = base_url + endpoint_description
        response = session.get(url, params=parameters)
        data = json.loads(response.text)["data"]
        return data
    except (ConnectionError, Timeout, TooManyRedirects) as e:
        print("error ", e) 

def get_coin_map(active):
    endpoint_map = "cryptocurrency/map"    
    parameters = {
        'listing_status': active
    }
    headers = {
        'Accepts': 'application/json',
        'X-CMC_PRO_API_KEY': cmc_key,
    }

    session = Session()
    session.headers.update(headers)

    try:
        url = base_url + endpoint_map
        response = session.get(url, params=parameters)
        data = json.loads(response.text)["data"]
        return data
    except (ConnectionError, Timeout, TooManyRedirects) as e:
        print(e) 

def exchange_map():    
    endpoint_map = "exchange/map"
    parameters = {
        #'listing_status': active
    }
    #limit
    #slug
    headers = {
        'Accepts': 'application/json',
        'X-CMC_PRO_API_KEY': cmc_key,
    }

    session = Session()
    session.headers.update(headers)

    try:
        url = base_url + endpoint_map
        response = session.get(url, params=parameters)
        data = json.loads(response.text)
        return data
    except (ConnectionError, Timeout, TooManyRedirects) as e:
        print(e) 

def _detect_by_application(self):
        try:
            session = self.request_handler.get_new_session()
            response = session.get(
                timeout=20,
                allow_redirects=True,
                url="{}://{}:{}".format(
                    self.host.protocol,
                    self.host.target,
                    self.host.port
                )
            )
            for waf, method in self.waf_app_method_map.items():
                result = method(response)
                if result:
                    self._waf_detected(waf, "web application")

        except (ConnectionError, TooManyRedirects) as e:
            raise WAFException("Couldn't get response from server.\n"
                               "Caused due to exception: {}".format(str(e))) 

def test_get_should_invalid_cache_on_too_many_redirects_error(mocker):
    delete_cache = mocker.patch("cachecontrol.caches.file_cache.FileCache.delete")

    response = Response()
    response.encoding = "utf-8"
    response.raw = BytesIO(encode('{"foo": "bar"}'))
    mocker.patch(
        "cachecontrol.adapter.CacheControlAdapter.send",
        side_effect=[TooManyRedirects(), response],
    )
    repository = PyPiRepository()
    repository._get("https://pypi.org/pypi/async-timeout/json")

    assert delete_cache.called 

def _get(self, endpoint):  # type: (str) -> Union[dict, None]
        try:
            json_response = self._session.get(self._base_url + endpoint)
        except TooManyRedirects:
            # Cache control redirect loop.
            # We try to remove the cache and try again
            self._cache_control_cache.delete(self._base_url + endpoint)
            json_response = self._session.get(self._base_url + endpoint)

        if json_response.status_code == 404:
            return None

        json_data = json_response.json()

        return json_data 

def http(self, method, host, payload=''):
    print 'HTTP %s %s: %s' % (method, host, payload)
    methodfn = getattr(requests, method.lower(), None)
    if method is None:
      raise NotImplementedError("requests module has not method %s " % method)
    try:
      if payload != '':
        request = methodfn(url='%s/%s' % (self.hosturl, host), data=json.dumps(payload), auth=self.auth, verify=False)
      else:
        request = methodfn(url='%s/%s' % (self.hosturl, host), auth=self.auth, verify=False)

      if request.status_code != requests.codes.ok:
        request.raise_for_status()

      rc = 0
      out = json.loads(request.text)
      err = ''
    except (ConnectionError, HTTPError, Timeout, TooManyRedirects) as e:
      rc = 1
      out = ''
      err = '%s. Error received: %s.\n Sent request: %s' % (
          e.message, json.loads(request.text), 'HTTP %s %s: %s' % (method, host, payload))

    print 'HTTP %s returned: %s' % (method, request.text)

    return (rc, out, err) 

def http(self, method, uri, payload=''):
    payload_str = json.dumps(payload)
    methodfn = getattr(requests, method.lower(), None)
    
    if method is None:
      raise NotImplementedError("requests module has not method %s " % method)
    try:
      if payload:
        request = methodfn(url='%s/%s' % (self.hosturl, uri), data=payload_str, auth=self.auth, verify=False)
      else:
        request = methodfn(url='%s/%s' % (self.hosturl, uri), auth=self.auth, verify=False)

      if request.status_code != requests.codes.ok:
        request.raise_for_status()

      rc = 0
      out = json.loads(request.text)
      err = ''
    except (ConnectionError, HTTPError, Timeout, TooManyRedirects) as e:
      rc = 1
      out = ""
      err = "Error sent request: HTTP %s %s: %s\n. Received error: %s" % (method, uri, payload_str,
        getattr(request, "text", ""))


    return (rc, out, err) 

def fetch(self, path, params=None):
        try:
            response = self.session.get(
                    '{}{}'.format(self.PRODUCTION_BASE_URL, path),
                    params=params)
            data = json.loads(response.text)
            return data
        except (ConnectionError, Timeout, TooManyRedirects) as e:
            self.logger.log(e) 

def send(self, method="GET", *args, **kwargs):
        """
        Send a GET/POST/HEAD request using the object's proxies and headers
        :param method: Method to send request in. GET/POST/HEAD
        """
        proxies = self._get_request_proxies()

        try:
            if method.upper() in self.allowed_methods:
                kwargs['timeout'] = kwargs['timeout'] if 'timeout' in kwargs else 5
                return request(method, proxies=proxies, headers=self.headers, cookies=self.cookies, *args, **kwargs)
            else:
                raise RequestHandlerException("Unsupported method: {}".format(method))
        except ProxyError:
            # TODO: Apply fail over for bad proxies or drop them
            raise RequestHandlerException("Error connecting to proxy")
        except (ConnectTimeout, ReadTimeout):
            raise RequestHandlerException("Connection with server timed out")
        except NewConnectionError:
            raise RequestHandlerException("Address cannot be resolved")
            # New connection error == Can't resolve address
        except ConnectionError:
            # TODO: Increase delay
            raise RequestHandlerException("Error connecting to host")
        except TooManyRedirects:
            raise RequestHandlerException("Infinite redirects detected - too many redirects error")
        except UnicodeDecodeError:
            # Following issue #19, apparently some sites do not use utf-8 in their uris :<>
            pass 

def get_web_application_info(self):
        session = self.request_handler.get_new_session()
        try:
            with session:
                # Test if target is serving HTTP requests
                response = session.get(
                    timeout=20,
                    url="{}://{}:{}".format(
                        self.host.protocol,
                        self.host.target,
                        self.host.port
                    )
                )
                self.headers = response.headers
                self._detect_cms()
                self._robots()
                self._sitemap()
                self._server_info()
                self._x_powered_by()
                self._cors_wildcard()
                self._xss_protection()
                self._anti_clickjacking()
                self._cookie_info(session.cookies)

                soup = BeautifulSoup(response.text, "lxml")
                self._find_urls(soup)
                self._find_forms(soup)
                self.storage_explorer.run(soup)

        except (ConnectionError, TooManyRedirects) as e:
            raise WebAppScannerException("Couldn't get response from server.\n"
                                         "Caused due to exception: {}".format(str(e))) 

def __call__(self, **kwargs):
        self._settings = kwargs
        self._proxy = {
            'http': kwargs['proxy'],
            'https': kwargs['proxy'],
        } if 'proxy' in kwargs else None
        requests_per_minute = kwargs['requests_per_minute']
        self._min_request_interval = 60 / requests_per_minute
        self._local_object_duration = kwargs['local_object_duration']
        self._broadcast_incremental_backup = kwargs['broadcast_incremental_backup']
        self._image_local_cache = kwargs['image_local_cache']
        self._broadcast_active_duration = kwargs['broadcast_active_duration']
        self._last_request_at = time()
        session = requests.Session()
        session.headers.update({
            'Cookie': self._account.session,
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.105 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'Referer': 'https://www.douban.com/',
            'Pragma': 'no-cache',
            'Cache-Control': 'no-cache',
        })
        self._request_session = session

        cookie = cookies.SimpleCookie()
        cookie.load(self._account.session)
        self._account_cookie = cookie

        try:
            return self.run()
        except (TooManyRedirects, Forbidden):
            logging.debug('Login session maybe forbidden')
            self._account.is_invalid = True
            self._account.save()
            return False
        #except Exception as e:
        #    logging.debug(e)
        #    return False
        finally:
            session.close() 

def request3(self, url, method="post", stream=False, data=None):
        self.last_url = url
        self.last_data = None
        self.r = None
        self.response = None
        self.last_error_message = "Not yet set by request3 function"
        if data is None:
            data = dict()
        data_json = json.dumps(data)
        headers = {
            "X-API-TOKEN": self.token,
            "Content-Type": "application/json"
        }
        try:
            if method == "post":
                self.last_data = data
                r = requests.post(url, data=data_json, headers=headers)
            elif method == "get":
                r = requests.get(url, headers=headers)
            else:
                raise NotImplementedError("method %s is not supported" % method)
        except (ConnectionError, Timeout, TooManyRedirects, HTTPError) as e:
            # http://docs.python-requests.org/en/master/user/quickstart/#errors-and-exceptions
            # ConnectionError: In the event of a network problem (e.g. DNS failure, refused connection, etc) Requests will raise a ConnectionError exception.
            # HTTPError: Response.raise_for_status() will raise an HTTPError if the HTTP request returned an unsuccessful status code.
            # Timeout: If a request times out, a Timeout exception is raised.
            # TooManyRedirects: If a request exceeds the configured number of maximum redirections, a TooManyRedirects exception is raised.
            self.last_error_message = str(e)
            return None
        self.r = r
        self.response = r.text   # Keep this for backward compatibility with previous versions
        try:
            self.json_response = r.json()
        except:
            self.json_response = None
        if r.status_code != 200:
            # HTTP server error: 404, 500 etc
            # Apparently http code 401 Unauthorized is returned when incorrect token is provided
            self.last_error_message = "HTTP Code %s" % r.status_code
            try:
                if "error" in self.json_response["meta"]:
                    self.last_error_message = self.json_response["meta"]["error"]["errorMessage"]
                    return None
            except:
                # Mailformed response from the server
                pass
            return None

        return r 

def generate_proxied_request(self, url, method="GET", params={}, data={}, headers={}, req_timeout=30):
        try:
            random.shuffle(self.proxy_list)
            # req_headers = dict(params.items() + self.generate_random_request_headers().items())

            req_headers = dict(params.items())
            req_headers_random = dict(self.generate_random_request_headers().items())
            req_headers.update(req_headers_random)

            if not self.sustain:
                self.randomize_proxy()

            headers.update(req_headers)

            self.logger.debug("Using headers: {0}".format(str(headers)))
            self.logger.debug("Using proxy: {0}".format(str(self.current_proxy)))
            request = requests.request(method, url, headers=headers, data=data, params=params, timeout=req_timeout,
                    proxies={"http": self.current_proxy.get_address(), "https": self.current_proxy.get_address()})
            # Avoid HTTP request errors
            if request.status_code == 409:
                raise ConnectionError("HTTP Response [409] - Possible Cloudflare DNS resolution error")
            elif request.status_code == 403:
                raise ConnectionError("HTTP Response [403] - Permission denied error")
            elif request.status_code == 503:
                raise ConnectionError("HTTP Response [503] - Service unavailable error")
            self.logger.info('RR Status {}'.format(request.status_code))
            return request
        except ConnectionError:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Proxy unreachable - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()
        except ReadTimeout:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Read timed out - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()
        except ChunkedEncodingError:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Wrong server chunked encoding - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()
        except TooManyRedirects:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Too many redirects - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()