Python requests.exceptions.ProxyError() Examples

def test_neutron_exception_is_raised_on_any_request_error(self):
        # timeout exception raises InfobloxTimeoutError
        f = mock.Mock()
        f.__name__ = 'mock'
        f.side_effect = req_exc.Timeout
        self.assertRaises(exceptions.InfobloxTimeoutError,
                          connector.reraise_neutron_exception(f))

        # all other request exception raises InfobloxConnectionError
        supported_exceptions = [req_exc.HTTPError,
                                req_exc.ConnectionError,
                                req_exc.ProxyError,
                                req_exc.SSLError,
                                req_exc.TooManyRedirects,
                                req_exc.InvalidURL]

        for ex in supported_exceptions:
            f.side_effect = ex
            self.assertRaises(exceptions.InfobloxConnectionError,
                              connector.reraise_neutron_exception(f)) 

def test_no_proxy_domain_fail(self, socks5_proxy):
        instance = {'proxy': {'http': 'http://1.2.3.4:567', 'no_proxy': '.google.com,example.com,example,9'}}
        init_config = {}
        http = RequestsWrapper(instance, init_config)

        # no_proxy not match: .google.com
        # ".y.com" matches "x.y.com" but not "y.com"
        with pytest.raises((ConnectTimeout, ProxyError)):
            http.get('http://google.com', timeout=1)

        # no_proxy not match: example or example.com
        with pytest.raises((ConnectTimeout, ProxyError)):
            http.get('http://notexample.com', timeout=1)

        with pytest.raises((ConnectTimeout, ProxyError)):
            http.get('http://example.org', timeout=1)

        # no_proxy not match: 9
        with pytest.raises((ConnectTimeout, ProxyError)):
            http.get('http://127.0.0.99', timeout=1) 

def get_url(url):
    headers['Referer'] = url
    count = 0
    while True:
        count += 1
        if count < settings['maxtries']:
            proxy = get_proxy()
        else:
            proxy = None
        try:
            resp = request('get', url, headers=headers, proxies={'http': proxy})
            return resp
        except ProxyError:
            if count > settings['maxtries']+2:
                print('Exit: Can not get url.<@get_url>')
                exit(1)
            continue 

def get_url(url):
    headers['Referer'] = url
    count = 0
    while True:
        count += 1
        if count < settings['maxtries']:
            proxy = get_proxy()
        else:
            proxy = None
        try:
            resp = request('get', url, headers=headers, proxies={'http': proxy})
            return resp
        except ProxyError:
            if count > settings['maxtries']+2:
                print('Exit: Could not get url.<@get_url>')
                exit(1)
            continue 

def exception_handle(method):
    """Handle exception raised by requests library."""

    def wrapper(*args, **kwargs):
        try:
            result = method(*args, **kwargs)
            return result
        except ProxyError:
            LOG.exception('ProxyError when try to get %s.', args)
            raise ProxyError('A proxy error occurred.')
        except ConnectionException:
            LOG.exception('ConnectionError when try to get %s.', args)
            raise ConnectionException('DNS failure, refused connection, etc.')
        except Timeout:
            LOG.exception('Timeout when try to get %s', args)
            raise Timeout('The request timed out.')
        except RequestException:
            LOG.exception('RequestException when try to get %s.', args)
            raise RequestException('Please check out your network.')

    return wrapper 

def test_proxy_env_vars_override_skip_fail(self):
        instance = {'skip_proxy': True}
        init_config = {}
        http = RequestsWrapper(instance, init_config)

        with EnvVars({'HTTP_PROXY': 'http://1.2.3.4:567'}):
            with pytest.raises((ConnectTimeout, ProxyError)):
                http.get('http://www.google.com', timeout=1, proxies=None)

        with EnvVars({'HTTPS_PROXY': 'https://1.2.3.4:567'}):
            with pytest.raises((ConnectTimeout, ProxyError)):
                http.get('https://www.google.com', timeout=1, proxies=None) 

def test_https_proxy_fail():
    old_env = dict(os.environ)
    os.environ['HTTPS_PROXY'] = BAD_PROXY_SETTINGS['https']

    try:
        with mock.patch('datadog_checks.checks.AgentCheck._get_requests_proxy', return_value={}):
            check = AgentCheck()
            proxies = check.get_instance_proxy({}, 'uri/health')
        with pytest.raises((ConnectTimeout, ProxyError)):
            requests.get('https://google.com', timeout=1, proxies=proxies)
    finally:
        os.environ.clear()
        os.environ.update(old_env) 

def test_http_proxy_fail():
    old_env = dict(os.environ)
    os.environ['HTTP_PROXY'] = BAD_PROXY_SETTINGS['http']

    try:
        with mock.patch('datadog_checks.checks.AgentCheck._get_requests_proxy', return_value={}):
            check = AgentCheck()
            proxies = check.get_instance_proxy({}, 'uri/health')
        with pytest.raises((ConnectTimeout, ProxyError)):
            requests.get('http://google.com', timeout=1, proxies=proxies)
    finally:
        os.environ.clear()
        os.environ.update(old_env) 

def send(self, method="GET", *args, **kwargs):
        """
        Send a GET/POST/HEAD request using the object's proxies and headers
        :param method: Method to send request in. GET/POST/HEAD
        """
        proxies = self._get_request_proxies()

        try:
            if method.upper() in self.allowed_methods:
                kwargs['timeout'] = kwargs['timeout'] if 'timeout' in kwargs else 5
                return request(method, proxies=proxies, headers=self.headers, cookies=self.cookies, *args, **kwargs)
            else:
                raise RequestHandlerException("Unsupported method: {}".format(method))
        except ProxyError:
            # TODO: Apply fail over for bad proxies or drop them
            raise RequestHandlerException("Error connecting to proxy")
        except (ConnectTimeout, ReadTimeout):
            raise RequestHandlerException("Connection with server timed out")
        except NewConnectionError:
            raise RequestHandlerException("Address cannot be resolved")
            # New connection error == Can't resolve address
        except ConnectionError:
            # TODO: Increase delay
            raise RequestHandlerException("Error connecting to host")
        except TooManyRedirects:
            raise RequestHandlerException("Infinite redirects detected - too many redirects error")
        except UnicodeDecodeError:
            # Following issue #19, apparently some sites do not use utf-8 in their uris :<>
            pass 

def retry_method(self, method, *args, **kwargs):
        if self.proxies:
            # fixme: may be a little loud
            logger.debug("Using proxy %s for: %s", self.proxies["http"], args[0])

        return retry_call(getattr(super(RetryingSession, self), method), fargs=args, fkwargs=kwargs, tries=3, delay=5,
                          exceptions=(exceptions.ConnectionError,
                                      exceptions.ProxyError,
                                      exceptions.SSLError,
                                      exceptions.Timeout,
                                      exceptions.ConnectTimeout,
                                      exceptions.ReadTimeout,
                                      socket.timeout)) 

def test_proxy_bad(self):
        instance = {'proxy': {'http': 'http://1.2.3.4:567', 'https': 'https://1.2.3.4:567'}}
        init_config = {}
        http = RequestsWrapper(instance, init_config)

        with pytest.raises((ConnectTimeout, ProxyError)):
            http.get('http://www.google.com', timeout=1)

        with pytest.raises((ConnectTimeout, ProxyError)):
            http.get('https://www.google.com', timeout=1) 

def check_and_save(self, proxy):
        self.headers['Referer'] = self.query_address
        try:
            resp = requests.get(self.query_address, proxies={'http': proxy}, headers=self.headers)
            html = PyQuery(resp.content.decode())
        except ProxyError:
            print('Expired:', proxy)
            return
        except UnicodeDecodeError:
            return
        result = html('code').eq(0).text()
        if result != self.primary_ip:
            self.redis_handler.sadd(r_server['s_name'], proxy) 

def test_invalid_proxy_address_versionclient(self, qe_token, qe_url):
        """Should raise RequestApiError with ProxyError using VersionClient."""
        # pylint: disable=unused-argument
        with self.assertRaises(RequestsApiError) as context_manager:
            version_finder = VersionClient(qe_url,
                                           proxies=INVALID_ADDRESS_PROXIES)
            version_finder.version()

        self.assertIsInstance(context_manager.exception.__cause__, ProxyError) 

def properties(self):
        """Return defined API properties for the current object."""
        if self._properties is None:
            try:
                r = self.tcex.session.options(self.api_endpoint, params={'show': 'readOnly'})
                if r.ok:
                    self._properties = r.json()
            except (ConnectionError, ProxyError):
                self.tcex.handle_error(
                    951, ['OPTIONS', 407, '{\"message\": \"Connection Error\"}', self.api_endpoint]
                )
        return self._properties 

def test_invalid_proxy_address_authclient(self, qe_token, qe_url):
        """Should raise RequestApiError with ProxyError using AuthClient."""
        with self.assertRaises(RequestsApiError) as context_manager:
            _ = AuthClient(qe_token, qe_url, proxies=INVALID_ADDRESS_PROXIES)

        self.assertIsInstance(context_manager.exception.__cause__, ProxyError) 

def test_invalid_proxy_port_versionclient(self, qe_token, qe_url):
        """Should raise RequestApiError with ProxyError using VersionClient."""
        with self.assertRaises(RequestsApiError) as context_manager:
            version_finder = VersionClient(qe_url, proxies=INVALID_PORT_PROXIES)
            version_finder.version()

        self.assertIsInstance(context_manager.exception.__cause__, ProxyError) 

def test_invalid_proxy_port_authclient(self, qe_token, qe_url):
        """Should raise RequestApiError with ProxyError using AuthClient."""
        with self.assertRaises(RequestsApiError) as context_manager:
            _ = AuthClient(qe_token, qe_url, proxies=INVALID_PORT_PROXIES)

        self.assertIsInstance(context_manager.exception.__cause__, ProxyError)

    # pylint: disable=unused-argument 

def test_failure_with_wrong_proxy(self):
        connection_policy.ProxyConfiguration.Port = self.serverPort + 1
        try:
            # client does a getDatabaseAccount on initialization, which fails
            client = cosmos_client.CosmosClient(self.host, {'masterKey': self.masterKey}, connection_policy)
            self.fail("Client instantiation is not expected")
        except Exception as e:
            self.assertTrue(type(e) is ProxyError, msg="Error is not a ProxyError") 

def test_proxy_error(self):
        signal = http_checks.check_fail(rex.ProxyError())
        self._assert_has_tags(self.conn_fail_tags, signal)
        self._assert_has_slug("HTTP_FAIL_PROXY_ERROR", signal) 

def _get_member_by_mid(self, mid: int) -> Optional[dict]:
        """
        根据用户id获取其信息
        :param mid: B站用户id
        :return: 用户详情 or None
        """
        get_params = {
            'mid': mid,
            'jsonp': 'jsonp'
        }
        try:
            res_json = requests.get(API_MEMBER_INFO, params=get_params, timeout=WAIT_MAX, proxies=self.cur_proxy,
                                    headers=self.headers).json()
        except ConnectTimeout as e:
            print(f'获取用户id: {mid} 详情失败: 请求接口超时, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ReadTimeout as e:
            print(f'获取用户id: {mid} 详情失败: 接口读取超时, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ValueError as e:
            # 解析json失败基本上就是ip被封了
            print(f'获取用户id: {mid} 详情失败: 解析json出错, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ProxyError as e:
            print(f'获取用户id: {mid} 详情失败: 连接代理失败, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except requests.ConnectionError as e:
            # 可以断定就是代理IP地址无效
            print(f'获取用户id: {mid} 详情失败: 连接错误, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ChunkedEncodingError as e:
            print(f'获取用户id: {mid} 详情失败: 远程主机强迫关闭了一个现有的连接, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        else:
            if res_json['code'] == -404:
                print(f'找不到用户mid:{mid}')
                raise UserNotFoundException(f'找不到用户mid:{mid}')
            if 'data' in res_json:
                return res_json['data']
            print(f'获取用户id: {mid} 详情失败: data字段不存在!')
        return 

def default_proxy_fail_exception_filter(req_exc):
    return isinstance(req_exc, (ProxyError, ConnectTimeout)) 

def test_pac_no_failover_available_exc_case(self):
        """Special case where proxy fails but there's no DIRECT fallback. Error should bubble up,
        and all applicable proxies should be tried again in the next request. Proxy failure from exception."""
        sess = PACSession(pac=PACFile(proxy_pac_js_tpl % 'PROXY a:80; PROXY b:80'))
        for _ in range(2):
            with _patch_request_base(side_effect=ProxyError()) as request, \
                    pytest.raises(ProxyError):
                sess.get(arbitrary_url)
            request.assert_has_calls([
                get_call(arbitrary_url, 'http://a:80'),
                get_call(arbitrary_url, 'http://b:80'),
            ]) 

def test_pac_failover_to_direct(self):
        """Proxy fails. Next in line is DIRECT keyword."""
        sess = PACSession(pac=PACFile(proxy_pac_js))

        def fake_request_reject_proxy(method, url, proxies=None, **kwargs):
            if proxies and proxies['http'] is not None:
                raise ProxyError()

        with _patch_request_base(side_effect=fake_request_reject_proxy) as request:
            sess.get(arbitrary_url)
            request.assert_has_calls([
                get_call(arbitrary_url, fake_proxy_url),
                get_call(arbitrary_url, 'DIRECT'),
            ]) 

def test_pac_failover(self):
        """First proxy raises error. Transparently fail over to second proxy."""
        sess = PACSession(pac=PACFile(proxy_pac_js_tpl % 'PROXY a:80; PROXY b:80; DIRECT'))

        def fake_request(method, url, proxies=None, **kwargs):
            if proxies and proxies['http'] == 'http://a:80':
                raise ProxyError()

        with _patch_request_base(side_effect=fake_request) as request:
            sess.get(arbitrary_url)
            request.assert_has_calls([
                get_call(arbitrary_url, 'http://a:80'),
                get_call(arbitrary_url, 'http://b:80'),
            ]) 

def test_bad_proxy_no_failover(self, proxy_host):
        """Verify that Requests returns ProxyError when given a non-existent proxy."""
        sess = PACSession(pac=PACFile(proxy_pac_js_tpl % 'PROXY %s:80' % proxy_host))
        with pytest.raises(ProxyError):
            sess.get(arbitrary_url) 

def test_timeout_proxy(self):
        # Travis can refuse quickly, and trigger ProxyError instead.
        session = requests.Session()
        with pytest.raises(ConnectTimeout):
            session.get(arbitrary_url, timeout=0.001, proxies=proxy_parameter_for_requests('http://localhost')) 

def test_unreachable_proxy(self, proxy_host):
        session = requests.Session()
        with pytest.raises(ProxyError):
            session.get(arbitrary_url, proxies=proxy_parameter_for_requests('http://' + proxy_host)) 

def delete(self):
        """Delete the Case Management Object.

        If no id is present in the obj then returns immediately.
        """
        if not self.id:  # pragma: no cover
            self.tcex.log.warning('A case without an ID cannot be deleted.')
            return

        url = f'{self.api_endpoint}/{self.id}'
        r = None
        try:
            r = self.tcex.session.delete(url)
            self.tcex.log.debug(
                f'Method: ({r.request.method.upper()}), '
                f'Status Code: {r.status_code}, '
                f'URl: ({r.url})'
            )
        except (ConnectionError, ProxyError):  # pragma: no cover
            self.tcex.handle_error(
                951, ['OPTIONS', 407, '{\"message\": \"Connection Error\"}', self.api_endpoint]
            )
        if len(r.content) < 5000:
            self.tcex.log.debug(u'response text: {}'.format(r.text))
        else:  # pragma: no cover
            self.tcex.log.debug(u'response text: (text to large to log)')
        if not self.success(r):
            err = r.text or r.reason
            if r.status_code == 404:
                self.tcex.handle_error(952, [r.request.method.upper(), r.status_code, err, r.url])
            self.tcex.handle_error(950, [r.status_code, err, r.url])
        return 

def test_server(self, requests):
        """Ensure it's possible to fetch cookies from a server and errors are handled properly"""
        requests.return_value = requests
        requests.get.return_value = requests

        requests.cookies = {'session': self.encoded}
        stdout, stderr = self.call('--decode', '--server', 'http://localhost:5000')
        self.assertEqual(stdout.read().strip(), str(self.decoded))

        requests.cookies = {'something-else': self.encoded}
        stdout, stderr = self.call('--decode', '--server', 'http://localhost:5000')
        self.assertNotEqual(stderr.read(), '', msg='Expected an error message')
        self.assertEqual(stdout.read(), '')

        requests.cookies = {'something-else': self.encoded}
        stdout, stderr = self.call('--decode', '--server', 'http://localhost:5000', '--cookie-name', 'something-else')
        self.assertEqual(stdout.read().strip(), str(self.decoded))

        requests.cookies = {'session': self.encoded}
        stdout, stderr = self.call(
            '--decode',
            '--server', 'http://localhost:5000',
            '--proxy', 'https://root:password@localhost:8080')

        self.assertEqual(stdout.read().strip(), str(self.decoded))

        for call in requests.mock_calls:
            if call[INDEX_FN_NAME] == 'get':
                if 'proxies' in call[INDEX_KWARGS]:
                    break

        else:
            raise AssertionError('Didn\'t find "proxies" argument in call args.')

        error_reason = ProxyError()
        error_reason.args = ('Cannot connect to proxy', OSError('Tunnel connection failed'))
        error = ProxyError(MaxRetryError(reason=error_reason, pool=MagicMock(), url='http://localhost:5000'))
        requests.get.side_effect = error

        requests.cookies = {'session': self.encoded}
        stdout, stderr = self.call(
            '--decode',
            '--server', 'http://localhost:5000',
            '--proxy', 'https://root:password@localhost:8080')

        self.assertIn('Tunnel connection failed', stderr.read().strip()) 

def check_fail(exception):
    """Checks for a requestslib exception, returns a signal if found.

    If this Exception is an instance of
    :class:`requests.exceptions.RequestException`, determine what kind of
    exception was raised. If not, return the results of from_generic_exception.

    :param Exception exception: An Exception object
    :returns: Signal with exception details
    :rtype: :class:`syntribos.signal.SynSignal`
    """
    check_name = "HTTP_CHECK_FAIL"

    def uncamel(string):
        string = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", string)
        return re.sub("([a-z0-9])([A-Z])", r"\1_\2", string).upper()

    if not isinstance(exception, rex.RequestException):
        return syntribos.signal.from_generic_exception(exception)

    data = {
        "response": exception.response,
        "request": exception.request,
        "exception": exception,
        "exception_name": uncamel(exception.__class__.__name__)
    }
    text = "An exception was encountered when sending the request. {desc}"
    slug = "HTTP_FAIL_{exc}".format(exc=data["exception_name"])
    tags = set(["EXCEPTION_RAISED"])

    invalid_request_exceptions = (rex.URLRequired, rex.MissingSchema,
                                  rex.InvalidSchema, rex.InvalidURL)

    if exception.__doc__:
        text = text.format(desc=exception.__doc__)
    else:
        text = text.format(
            desc="An unknown exception was raised. Please report this.")

    # CONNECTION FAILURES
    if isinstance(exception, (rex.ProxyError, rex.SSLError,
                              rex.ChunkedEncodingError, rex.ConnectionError)):
        tags.update(["CONNECTION_FAIL"])
    # TIMEOUTS
    elif isinstance(exception, (rex.ConnectTimeout, rex.ReadTimeout)):
        tags.update(["CONNECTION_TIMEOUT", "SERVER_FAIL"])
    # INVALID REQUESTS
    elif isinstance(exception, invalid_request_exceptions):
        tags.update(["INVALID_REQUEST", "CLIENT_FAIL"])

    return syntribos.signal.SynSignal(
        text=text,
        slug=slug,
        strength=1.0,
        tags=list(tags),
        data=data,
        check_name=check_name)