Python urllib3.Retry() Examples

def post_with_retries(url: str, data: dict, retries: int, backoff: float) -> int:
    """
    Make a POST request with retries.

    >>> post_with_retries('http://httpstat.us/503', {}, retries=1, backoff=0)
    500
    >>> post_with_retries('https://httpstat.us/200', {}, retries=1, backoff=0)
    200
    """
    retry_adapter = HTTPAdapter(max_retries=Retry(
        total=retries,
        backoff_factor=backoff,
        status_forcelist=[500, 502, 503, 504],
        method_whitelist=frozenset(['POST'])
    ))

    with Session() as session:
        session.mount('http://', retry_adapter)
        session.mount('https://', retry_adapter)

        try:
            response = session.post(url, data=data)
        except RetryError:
            return 500

        return response.status_code 

def __init__(self, tor_controller=None):
        if not self.__socket_is_patched():
            gevent.monkey.patch_socket()
        self.tor_controller = tor_controller
        if not self.tor_controller:
            retries = urllib3.Retry(35)
            user_agent = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}
            self.session = urllib3.PoolManager(maxsize=35,
                                               cert_reqs='CERT_REQUIRED',
                                               ca_certs=certifi.where(),
                                               headers=user_agent,
                                               retries=retries)
        else:
            self.session = self.tor_controller.get_tor_session()
        self.__tor_status__()
        self.languages = self._get_all_languages() 

def retry_session(self):
        if not hasattr(self, "_retry_session"):
            self._retry_session = requests.Session()
            retry = Retry(
                total=3,
                read=3,
                connect=3,
                backoff_factor=2,
                status_forcelist=[429, 500, 502, 503, 504],
            )
            adapter = HTTPAdapter(max_retries=retry)
            self._retry_session.mount("http://", adapter)
            self._retry_session.mount("https://", adapter)
            self._threaded_done = 0
            self._threaded_exceptions = 0
            self._periodic_http_done = 0
            self._periodic_http_exceptions = 0
            self._periodic_ws_done = 0
            self._periodic_ws_exceptions = 0
        return self._retry_session 

def __init__(self) -> None:
        http_client = urllib3.PoolManager(
            timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
            cert_reqs='CERT_REQUIRED',
            ca_certs=certifi.where(),
            retries=urllib3.Retry(
                total=5,
                backoff_factor=0.2,
                status_forcelist=[500, 502, 503, 504]
            ),
            maxsize=20
        )
        self.client = minio.Minio(
            S3_SERVER,
            access_key=S3_ACCESS_KEY,
            secret_key=S3_SECRET_KEY,
            region=S3_REGION,
            secure=S3_SERVER == 's3.amazonaws.com',
            http_client=http_client
        ) 

def retry_session(self):
        if not hasattr(self, '_retry_session'):
            self._retry_session = requests.Session()
            retry = Retry(
                total=3,
                read=3,
                connect=3,
                backoff_factor=2,
                status_forcelist=[429, 500, 502, 503, 504],
            )
            adapter = HTTPAdapter(max_retries=retry)
            self._retry_session.mount('http://', adapter)
            self._retry_session.mount('https://', adapter)
            self._threaded_done = 0
            self._threaded_exceptions = 0
            self._periodic_http_done = 0
            self._periodic_http_exceptions = 0
            self._periodic_ws_done = 0
            self._periodic_ws_exceptions = 0
        return self._retry_session 

def _conn_request(self, conn, request_uri, method, body, headers):
        full_uri = self._create_full_uri(conn, request_uri)

        decode = True if method != 'HEAD' else False

        try:
            urllib3_response = self.pool.request(
                method,
                full_uri,
                body=body,
                headers=headers,
                redirect=False,
                retries=urllib3.Retry(total=False, redirect=0),
                timeout=urllib3.Timeout(total=self.timeout),
                decode_content=decode)

            response = _map_response(urllib3_response, decode=decode)
            content = urllib3_response.data

        except Exception as e:
            raise _map_exception(e)

        return response, content 

def __init__(self, endpoint=None, http_client=None):
        self._endpoint = endpoint or "http://169.254.169.254"
        self._http_client = http_client or urllib3.PoolManager(
            retries=urllib3.Retry(
                total=5,
                backoff_factor=0.2,
                status_forcelist=[500, 502, 503, 504],
            ),
        ) 

def test_custom_retries(self):
        RETRIES = Retry(20, backoff_factor=0.1)
        options = {'path': 'test/', 'retries': RETRIES}

        self.factory_custom_proxy_view(**options)
        url = 'http://www.example.com/test/'
        headers = {'Cookie': ''}
        self.urlopen.assert_called_with('GET', url, redirect=False,
                                        retries=RETRIES,
                                        preload_content=False,
                                        decode_content=False,
                                        headers=headers, body=b'') 

def get_session(*args: Any, **kwargs: Any) -> "ForestSession":
    """
    Create a requests session to access the REST API

    :return: requests session
    :rtype: Session
    """
    session = ForestSession(*args, **kwargs)
    retry_adapter = HTTPAdapter(
        max_retries=Retry(
            total=3,
            method_whitelist=["POST"],
            status_forcelist=[502, 503, 504, 521, 523],
            backoff_factor=0.2,
            raise_on_status=False,
        )
    )

    session.mount("http://", retry_adapter)
    session.mount("https://", retry_adapter)

    # We need this to get binary payload for the wavefunction call.
    session.headers.update({"Accept": "application/octet-stream"})

    session.headers.update({"Content-Type": "application/json; charset=utf-8"})

    return session 

def get_session(self):
        session = requests.Session()
        retries = Retry(total=self.max_attempts,
                        backoff_factor=self.timeout / 1000,
                        method_whitelist=['POST'],
                        status_forcelist=[202])
        session.mount('https://', HTTPAdapter(max_retries=retries))
        return session 

def _build_session(self):
        self._session = requests.Session()
        retry = urllib3.Retry(
            total=self.retries,
            connect=None,
            read=False,
            status=self.status_retries,
            backoff_factor=self.backoff_factor,
            status_forcelist=self.status_forcelist)

        adapter = requests.adapters.HTTPAdapter(max_retries=retry)
        self._session.mount('http://', adapter)
        self._session.mount('https://', adapter) 

def __init__(self, endpoint, access_key=None,
                 secret_key=None,
                 session_token=None,
                 secure=True,
                 region=None,
                 http_client=None,
                 credentials=None):

        # Validate endpoint.
        is_valid_endpoint(endpoint)

        # Validate http client has correct base class.
        if http_client and not isinstance(
                http_client,
                urllib3.poolmanager.PoolManager):
            raise InvalidArgumentError(
                'HTTP client should be of instance'
                ' `urllib3.poolmanager.PoolManager`'
            )

        # Default is a secured connection.
        scheme = 'https://' if secure else 'http://'
        self._region = region or get_s3_region_from_endpoint(endpoint)
        self._region_map = dict()
        self._endpoint_url = urlsplit(scheme + endpoint).geturl()
        self._is_ssl = secure
        self._access_key = access_key
        self._secret_key = secret_key
        self._session_token = session_token
        self._user_agent = _DEFAULT_USER_AGENT
        self._trace_output_stream = None
        self._enable_s3_accelerate = False
        self._accelerate_endpoint_url = scheme + 's3-accelerate.amazonaws.com'
        self._credentials = credentials or Credentials(
            provider=Chain(
                providers=[
                    Static(access_key, secret_key, session_token),
                    EnvAWS(),
                    EnvMinio(),
                ]
            )
        )

        # Load CA certificates from SSL_CERT_FILE file if set
        ca_certs = os.environ.get('SSL_CERT_FILE') or certifi.where()
        self._http = http_client or urllib3.PoolManager(
            timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
            maxsize=MAX_POOL_SIZE,
            cert_reqs='CERT_REQUIRED',
            ca_certs=ca_certs,
            retries=urllib3.Retry(
                total=5,
                backoff_factor=0.2,
                status_forcelist=[500, 502, 503, 504]
            )
        )

    # Set application information.