Python requests.adapters() Examples

def __init__(self, endpoint, session_id=None, req_session=None, args=None):
        self._endpoint = endpoint.rstrip('/')
        self._session_id = session_id
        self._args = args or dict()
        # dict structure: {tileable_key -> graph_key, tileable_ids}
        # dict value is a tuple object which records graph key and tileable id
        self._executed_tileables = dict()

        self._serial_type = None
        self._pickle_protocol = pickle.HIGHEST_PROTOCOL

        if req_session:
            self._req_session = req_session
        else:
            import requests
            from requests.adapters import HTTPAdapter

            self._req_session = requests.Session()
            self._req_session.mount('http://stackoverflow.com', HTTPAdapter(max_retries=5))

        self._main() 

def _set_response_ins_(self, pageurl):
        """
        Sets the response for the GET request of pageurl and stores it in self.resp

        :param pageurl: url for which we store the response.
        """
        try:
            s = requests.Session()
            a = requests.adapters.HTTPAdapter(max_retries=5)
            s.mount('http://', a)
            resp = s.get(pageurl, timeout=30)
            self.__resp_obj__ = resp
            resp.close()
        except requests.exceptions.Timeout:
            logging.error("\tVery Slow Internet Connection.")
        except requests.exceptions.ConnectionError:
            logging.error("\tNetwork Unavailable. Check your connection.")
        except requests.exceptions.MissingSchema:
            logging.error("\t503 Service Unavailable. Retrying download ... ") 

def test_configure_mtls_channel_with_callback(self):
        mock_callback = mock.Mock()
        mock_callback.return_value = (
            pytest.public_cert_bytes,
            pytest.private_key_bytes,
        )

        auth_session = google.auth.transport.requests.AuthorizedSession(
            credentials=mock.Mock()
        )
        auth_session.configure_mtls_channel(mock_callback)

        assert auth_session.is_mtls
        assert isinstance(
            auth_session.adapters["https://"],
            google.auth.transport.requests._MutualTlsAdapter,
        ) 

def __init__(self, account, endpoint, user_agent=None, proxies=None, stream=False, retry_times=3, conn_timeout=5,
                 read_timeout=120, pool_connections=10, pool_maxsize=10, exception_handler_=exception_handler):
        if endpoint.endswith('/'):
            endpoint = endpoint[:-1]
        self._account = account
        self._endpoint = endpoint
        self._user_agent = user_agent or default_user_agent()
        self._proxies = proxies
        self._stream = stream
        self._retry_times = retry_times
        self._conn_timeout = conn_timeout
        self._read_timeout = read_timeout

        self._session = requests.Session()
        self._session.headers.update({Headers.ACCEPT_ENCODING: ''})

        # mount adapters with retry times
        adapter = HTTPAdapter(pool_connections=pool_connections, pool_maxsize=pool_maxsize,
                              max_retries=self._retry_times)
        self._session.mount('http://', adapter)
        self._session.mount('https://', adapter)

        # exception handler
        self._exception_handler = exception_handler_ 

def __init__(self, credentials,
                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
                 refresh_timeout=None,
                 **kwargs):
        super(AuthorizedSession, self).__init__(**kwargs)
        self.credentials = credentials
        self._refresh_status_codes = refresh_status_codes
        self._max_refresh_attempts = max_refresh_attempts
        self._refresh_timeout = refresh_timeout

        auth_request_session = requests.Session()

        # Using an adapter to make HTTP requests robust to network errors.
        # This adapter retrys HTTP requests when network errors occur
        # and the requests seems safely retryable.
        retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)
        auth_request_session.mount("https://", retry_adapter)

        # Request instance used by internal methods (for example,
        # credentials.refresh).
        # Do not pass `self` as the session here, as it can lead to infinite
        # recursion.
        self._auth_request = Request(auth_request_session) 

def use(
            cls,
            environment_or_domain: Union[Environment, str] = Environment.PROD,
            client_id: Optional[str] = None,
            client_secret: Optional[str] = None,
            scopes: Optional[Union[Tuple, List, str]] = (),
            api_version: str = API_VERSION,
            application: str = DEFAULT_APPLICATION,
            http_adapter: requests.adapters.HTTPAdapter = None
    ) -> None:
        environment_or_domain = environment_or_domain.name if isinstance(environment_or_domain,
                                                                         Environment) else environment_or_domain
        session = cls.get(
            environment_or_domain,
            client_id=client_id,
            client_secret=client_secret,
            scopes=scopes,
            api_version=api_version,
            application=application,
            http_adapter=http_adapter
        )

        session.init()
        cls.current = session 

def test_configure_mtls_channel_with_metadata(self, mock_get_client_cert_and_key):
        mock_get_client_cert_and_key.return_value = (
            True,
            pytest.public_cert_bytes,
            pytest.private_key_bytes,
        )

        auth_session = google.auth.transport.requests.AuthorizedSession(
            credentials=mock.Mock()
        )
        auth_session.configure_mtls_channel()

        assert auth_session.is_mtls
        assert isinstance(
            auth_session.adapters["https://"],
            google.auth.transport.requests._MutualTlsAdapter,
        ) 

def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK):
        """Initializes a urllib3 PoolManager. This method should not be called
        from user code, and is only exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param connections: The number of urllib3 connection pools to cache.
        :param maxsize: The maximum number of connections to save in the pool.
        :param block: Block when no free connections are available.
        """
        # save these values for pickling
        self._pool_connections = connections
        self._pool_maxsize = maxsize
        self._pool_block = block

        self.poolmanager = AsyncPoolManager(num_pools=connections, maxsize=maxsize, block=block)
        self.connections = [] 

def __init__(self):
        self._config_timestamp = None
        self._mode = None

        self.authorization_endpoint = None
        self.signing_keys = None
        self.token_endpoint = None
        self.end_session_endpoint = None
        self.issuer = None

        method_whitelist = frozenset([
            'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST'
        ])

        retry = Retry(
            total=settings.RETRIES,
            read=settings.RETRIES,
            connect=settings.RETRIES,
            backoff_factor=0.3,
            method_whitelist=method_whitelist
        )
        self.session = requests.Session()
        adapter = requests.adapters.HTTPAdapter(max_retries=retry)
        self.session.mount('https://', adapter)
        self.session.verify = settings.CA_BUNDLE 

def retryable_session(session: requests.Session, retries: int = 8) -> requests.Session:
    """
    Session with requests to allow for re-attempts at downloading missing data
    :param session: Session to download with
    :param retries: How many retries to attempt
    :return: Session that does downloading
    """
    retry = urllib3.util.retry.Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=0.3,
        status_forcelist=(500, 502, 504),
    )

    adapter = requests.adapters.HTTPAdapter(max_retries=retry)
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    return session 

def __init__(self, *args, **kwargs):
        self.assert_hostname = kwargs.pop('assert_hostname', True)
        requests.adapters.HTTPAdapter.__init__(self, *args, **kwargs) 

def __init__(self, url=None,
                 base_url=None,
                 proxy=None,
                 username=None,
                 password=None,
                 verify=False,
                 minutes=3,
                 timeout=30):
        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
        OpTestLogger.optest_logger_glob.setUpChildLogger("urllib3")
        self.username = username
        self.password = password
        self.session = requests.Session()
        if self.username is not None and self.password is not None:
            self.session.auth = (self.username, self.password)
        self.session.verify = verify
        self.jsonHeader = {'Content-Type': 'application/json'}
        self.xAuthHeader = {}
        self.timeout = timeout
        self.minutes = minutes
        self.session.mount('https://', HTTPAdapter(max_retries=5))
        # value.max_retries for future debug if needed
#        for key, value in self.session.adapters.items():
#            log.debug("max_retries={}".format(value.max_retries))

        if proxy:
            self.session.proxies = {"http": proxy}
        else:
            self.session.proxies = {}

        self.base_url = url + (base_url if base_url else "") 

def __init__(self, environment_or_domain: str, api_version: str = API_VERSION,
                     application: str = DEFAULT_APPLICATION, http_adapter: requests.adapters.HTTPAdapter = None):
            domain, verify = self.domain_and_verify(environment_or_domain)
            GsSession.__init__(self, domain, api_version=api_version, application=application, verify=verify,
                               http_adapter=http_adapter) 

def __init__(self, credentials,
                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
                 refresh_timeout=None,
                 auth_request=None):
        super(AuthorizedSession, self).__init__()
        self.credentials = credentials
        self._refresh_status_codes = refresh_status_codes
        self._max_refresh_attempts = max_refresh_attempts
        self._refresh_timeout = refresh_timeout

        if auth_request is None:
            auth_request_session = requests.Session()

            # Using an adapter to make HTTP requests robust to network errors.
            # This adapter retrys HTTP requests when network errors occur
            # and the requests seems safely retryable.
            retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)
            auth_request_session.mount("https://", retry_adapter)

            # Do not pass `self` as the session here, as it can lead to
            # infinite recursion.
            auth_request = Request(auth_request_session)

        # Request instance used by internal methods (for example,
        # credentials.refresh).
        self._auth_request = auth_request 

def get(
            cls,
            environment_or_domain: Union[Environment, str] = Environment.PROD,
            client_id: Optional[str] = None,
            client_secret: Optional[str] = None,
            scopes: Optional[Union[Tuple, List, str]] = (),
            token: str = '',
            is_gssso: bool = False,
            api_version: str = API_VERSION,
            application: str = DEFAULT_APPLICATION,
            http_adapter: requests.adapters.HTTPAdapter = None
    ) -> 'GsSession':
        """ Return an instance of the appropriate session type for the given credentials"""

        environment_or_domain = environment_or_domain.name if isinstance(environment_or_domain,
                                                                         Environment) else environment_or_domain

        if client_id is not None:
            if isinstance(scopes, str):
                scopes = (scopes,)

            scopes = tuple(set(itertools.chain(scopes, cls.Scopes.get_default())))

            return OAuth2Session(environment_or_domain, client_id, client_secret, scopes, api_version=api_version,
                                 application=application, http_adapter=http_adapter)
        elif token:
            if is_gssso:
                try:
                    return PassThroughGSSSOSession(environment_or_domain, token, api_version=api_version,
                                                   application=application, http_adapter=http_adapter)
                except NameError:
                    raise MqUninitialisedError('This option requires gs_quant_internal to be installed')
            else:
                return PassThroughSession(environment_or_domain, token, api_version=api_version,
                                          application=application, http_adapter=http_adapter)
        else:
            try:
                return KerberosSession(environment_or_domain, api_version=api_version, http_adapter=http_adapter)
            except NameError:
                raise MqUninitialisedError('Must specify client_id and client_secret') 

def __init__(self, domain: str, api_version: str = API_VERSION, application: str = DEFAULT_APPLICATION, verify=True,
                 http_adapter: requests.adapters.HTTPAdapter = None):
        super().__init__()
        self._session = None
        self.domain = domain
        self.api_version = api_version
        self.application = application
        self.verify = verify
        self.http_adapter = requests.adapters.HTTPAdapter(pool_maxsize=100) if http_adapter is None else http_adapter 

def _make_requests_session_with_retries(max_retries):
    from requests.adapters import HTTPAdapter
    from requests.packages.urllib3.util import Retry

    s = requests.Session()
    a = HTTPAdapter(
        max_retries=Retry(
            total=max_retries,
            status_forcelist=[  # this is a list of statuses to consider to be
                                # an error and retry.
                429,  # Too many requests (i.e: back off)
                500,  # Generic internal server error
                502,  # Bad Gateway - i.e: upstream failure
                503,  # Unavailable, temporarily
                504,  # Gateway timeout
                522   # Origin connection timed out
            ],
            backoff_factor=1.0  # back off for 0s, 1s, 3s, 7s, etc... after
                                # each successive failure. (factor*(2^N-1))
        ))

    # use retry for both HTTP and HTTPS connections.
    s.mount('http://', a)
    s.mount('https://', a)

    return s 

def __init__(self):
        super().__init__()
        self.session = requests.Session()
        a = requests.adapters.HTTPAdapter(
            max_retries=Retry(method_whitelist=frozenset(['GET', 'POST']))
        )
        self.session.mount('http://', a) 

def __init__(self):
        super().__init__()

        self.session = requests.Session()
        a = requests.adapters.HTTPAdapter(max_retries=10)
        self.session.mount('http://', a) 

def __init__(self):
        super().__init__()

        self.session = requests.Session()
        a = requests.adapters.HTTPAdapter(max_retries=10)
        self.session.mount('http://', a) 

def __init__(self):
        super().__init__()
        self.session = requests.Session()
        a = requests.adapters.HTTPAdapter(max_retries=10)
        self.session.mount('http://', a) 

def __init__(self, credentials,
                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
                 refresh_timeout=None,
                 auth_request=None):
        super(AuthorizedSession, self).__init__()
        self.credentials = credentials
        self._refresh_status_codes = refresh_status_codes
        self._max_refresh_attempts = max_refresh_attempts
        self._refresh_timeout = refresh_timeout

        if auth_request is None:
            auth_request_session = requests.Session()

            # Using an adapter to make HTTP requests robust to network errors.
            # This adapter retrys HTTP requests when network errors occur
            # and the requests seems safely retryable.
            retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)
            auth_request_session.mount("https://", retry_adapter)

            # Do not pass `self` as the session here, as it can lead to
            # infinite recursion.
            auth_request = Request(auth_request_session)

        # Request instance used by internal methods (for example,
        # credentials.refresh).
        self._auth_request = auth_request 

def __init__(self, credentials,
                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
                 refresh_timeout=None,
                 auth_request=None):
        super(AuthorizedSession, self).__init__()
        self.credentials = credentials
        self._refresh_status_codes = refresh_status_codes
        self._max_refresh_attempts = max_refresh_attempts
        self._refresh_timeout = refresh_timeout

        if auth_request is None:
            auth_request_session = requests.Session()

            # Using an adapter to make HTTP requests robust to network errors.
            # This adapter retrys HTTP requests when network errors occur
            # and the requests seems safely retryable.
            retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)
            auth_request_session.mount("https://", retry_adapter)

            # Do not pass `self` as the session here, as it can lead to
            # infinite recursion.
            auth_request = Request(auth_request_session)

        # Request instance used by internal methods (for example,
        # credentials.refresh).
        self._auth_request = auth_request 

def __init__(self, credentials,
                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
                 refresh_timeout=None,
                 auth_request=None):
        super(AuthorizedSession, self).__init__()
        self.credentials = credentials
        self._refresh_status_codes = refresh_status_codes
        self._max_refresh_attempts = max_refresh_attempts
        self._refresh_timeout = refresh_timeout

        if auth_request is None:
            auth_request_session = requests.Session()

            # Using an adapter to make HTTP requests robust to network errors.
            # This adapter retrys HTTP requests when network errors occur
            # and the requests seems safely retryable.
            retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)
            auth_request_session.mount("https://", retry_adapter)

            # Do not pass `self` as the session here, as it can lead to
            # infinite recursion.
            auth_request = Request(auth_request_session)

        # Request instance used by internal methods (for example,
        # credentials.refresh).
        self._auth_request = auth_request 

def __init__(self, credentials,
                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
                 refresh_timeout=None,
                 auth_request=None):
        super(AuthorizedSession, self).__init__()
        self.credentials = credentials
        self._refresh_status_codes = refresh_status_codes
        self._max_refresh_attempts = max_refresh_attempts
        self._refresh_timeout = refresh_timeout

        if auth_request is None:
            auth_request_session = requests.Session()

            # Using an adapter to make HTTP requests robust to network errors.
            # This adapter retrys HTTP requests when network errors occur
            # and the requests seems safely retryable.
            retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)
            auth_request_session.mount("https://", retry_adapter)

            # Do not pass `self` as the session here, as it can lead to
            # infinite recursion.
            auth_request = Request(auth_request_session)

        # Request instance used by internal methods (for example,
        # credentials.refresh).
        self._auth_request = auth_request 

def set_retry_on_busy(self, total=5, backoff_factor=0.1, status_forcelist=None, **kwargs):
        """
        Mount a custom retry object on the current session that allows service level
        retries when the SMC might reply with a Service Unavailable (503) message.
        This can be possible in larger environments with higher database activity.
        You can all this on the existing session, or provide as a dict to the login
        constructor.
        
        :param int total: total retries
        :param float backoff_factor: when to retry
        :param list status_forcelist: list of HTTP error codes to retry on
        :param list method_whitelist: list of methods to apply retries for, GET, POST and
            PUT by default
        :return: None
        """
        if self.session:
            from requests.adapters import HTTPAdapter
            from requests.packages.urllib3.util.retry import Retry
    
            method_whitelist = kwargs.pop('method_whitelist', []) or ['GET', 'POST', 'PUT']
            status_forcelist = frozenset(status_forcelist) if status_forcelist else frozenset([503])
            retry = Retry(
                total=total,
                backoff_factor=backoff_factor,
                status_forcelist=status_forcelist,
                method_whitelist=method_whitelist)
            
            for proto_str in ('http://', 'https://'):
                self.session.mount(proto_str, HTTPAdapter(max_retries=retry))
            logger.debug('Mounting retry object to HTTP session: %s' % retry) 

def __init__(
        self,
        credentials,
        refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
        max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
        refresh_timeout=None,
        auth_request=None,
    ):
        super(AuthorizedSession, self).__init__()
        self.credentials = credentials
        self._refresh_status_codes = refresh_status_codes
        self._max_refresh_attempts = max_refresh_attempts
        self._refresh_timeout = refresh_timeout
        self._is_mtls = False

        if auth_request is None:
            auth_request_session = requests.Session()

            # Using an adapter to make HTTP requests robust to network errors.
            # This adapter retrys HTTP requests when network errors occur
            # and the requests seems safely retryable.
            retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)
            auth_request_session.mount("https://", retry_adapter)

            # Do not pass `self` as the session here, as it can lead to
            # infinite recursion.
            auth_request = Request(auth_request_session)

        # Request instance used by internal methods (for example,
        # credentials.refresh).
        self._auth_request = auth_request 

def run(self):
        self._executor = ThreadPoolExecutor(self.concurrency)
        self.session = requests.Session()
        adapter = requests.adapters.HTTPAdapter(
            pool_connections=self.concurrency, pool_maxsize=self.concurrency)
        self.session.mount('http://', adapter)
        self.session.mount('https://', adapter)
        self.session.verify = self.verify_ssl

        t0 = time()
        last_report = time()
        i = 0
        r = None
        for r in self.perform_requests():
            if r is not True:
                i += 1
                self.ui.info('{} responses sent | time elapsed {}s'
                             .format(i, time() - t0))

                if time() - last_report > REPORT_INTERVAL:
                    self.progress_queue.put((
                        ProgressQueueMsg.NETWORK_PROGRESS, {
                            "processed": self.n_requests,
                            "retried": self.n_retried,
                            "consumed": self.n_consumed,
                            "rusage": get_rusage(),
                        }))
                    last_report = time()

        self.progress_queue.put((ProgressQueueMsg.NETWORK_DONE, {
            "ret": r,
            "processed": self.n_requests,
            "retried": self.n_retried,
            "consumed": self.n_consumed,
            "rusage": get_rusage(),
        })) 

def cancel(self):
        for v in self.adapters.values():
            v.close()
            v.cancel() 

def get_connection(self, url, proxies=None):
        """Returns a urllib3 connection for the given URL. This should not be
        called from user code, and is only exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param url: The URL to connect to.
        :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
        """
        proxies = proxies or {}
        proxy = proxies.get(urlparse(url.lower()).scheme)

        if proxy:
            proxy_headers = self.proxy_headers(proxy)

            if proxy not in self.proxy_manager:
                self.proxy_manager[proxy] = proxy_from_url(
                    proxy,
                    proxy_headers=proxy_headers,
                    num_pools=self._pool_connections,
                    maxsize=self._pool_maxsize,
                    block=self._pool_block
                )

            conn = self.proxy_manager[proxy].connection_from_url(url)
        else:
            # Only scheme should be lower case
            parsed = urlparse(url)
            url = parsed.geturl()
            conn = self.poolmanager.connection_from_url(url)

        self.connections.append(conn)
        return conn