Python requests_futures.sessions.FuturesSession() Examples

def _async_requests(urls):
    """
    Sends multiple non-blocking requests. Returns
    a list of responses.

    :param urls:
        List of urls
    """
    session = FuturesSession(max_workers=30)
    futures = [
        session.get(url)
        for url in urls
    ]
    return [ future.result() for future in futures ] 

def test_send_requests_worker(monkeypatch):
    load = [
        {
            "url": "http://shadowreader.example.com",
            "req_method": "POST",
            "request_uri": "/test",
        }
    ]
    headers = {"User-Agent": "sr_pytest"}

    fut = FuturesSession().get("http://www.example.com")
    monkeypatch.setattr("libs.worker._send_request", lambda a, b, c, d, e, f, g: fut)

    futs, timeouts, exceptions = worker.send_requests_worker(
        load=load, delay=0.1, random_delay=True, headers=headers
    )

    assert len(futs) == 1

    fut = futs[0]["fut"]
    assert fut.result().status_code == 200 

def send_ga_event(event, user):
    session = FuturesSession()
    payload = {
        "v": 1,
        "tid": settings.GOOGLE_TRACKING_ID,
        "t": "event",
        "ec": "email",
        "ea": event.event_type,
        "cm": "email",
    }
    if event.esp_event:
        payload["ua"] = event.esp_event.get("user-agent")
        payload["dt"] = event.esp_event.get("subject", [None])[0]
        payload["cn"] = event.esp_event.get("campaign_name", None)
        payload["cs"] = event.esp_event.get("campaign_source", None)
        payload["cc"] = payload["el"] = event.esp_event.get("email_id", None)
        payload["dp"] = "%s/%s" % (payload["cc"], event.event_type)
    else:
        logger.warn("No ESP event found for event: %s" % event.__dict__)
    logger.info("Sending mail event data Analytics: %s" % payload)
    session.post("https://www.google-analytics.com/collect", data=payload) 

def __init__(self, cache: bool = False, future: bool = True):
        if cache:
            redis_conn = redis.StrictRedis(host='redis')
            self.session = requests_cache.core.CachedSession(
                cache_name='api_cache',
                backend='redis', expire_after=60 * 60 * 24 * 30,
                allowable_codes=(200,),
                allowable_methods=('GET',),
                old_data_on_error=False,
                connection=redis_conn,
            )
        else:
            self.session = session()
        if future:
            self.future_session = FuturesSession(max_workers=10, session=self.session)
        self.url = self.url_template.format(resource='', token=self.token) 

def send_slos(queryset_list):
        """
            Send SLO requests to each ticket of each queryset of ``queryset_list``

            :param list queryset_list: A list a :class:`Ticket` queryset
            :return: A list of possibly encoutered :class:`Exception`
            :rtype: list
        """
        # sending SLO to timed-out validated tickets
        async_list = []
        session = FuturesSession(
            executor=ThreadPoolExecutor(max_workers=settings.CAS_SLO_MAX_PARALLEL_REQUESTS)
        )
        errors = []
        for queryset in queryset_list:
            for ticket in queryset:
                ticket.logout(session, async_list)
            queryset.delete()
        for future in async_list:
            if future:  # pragma: no branch (should always be true)
                try:
                    future.result()
                except Exception as error:
                    errors.append(error)
        return errors 

def _init_session(session, **kwargs):
    if session is None:
        if kwargs.get('asynchronous'):
            session = FuturesSession(max_workers=kwargs.get('max_workers', 8))
        else:
            session = Session()
        if kwargs.get('proxies'):
            session.proxies = kwargs.get('proxies')
        retries = Retry(
            total=3,
            backoff_factor=1,
            status_forcelist=[429, 500, 502, 503, 504],
            method_whitelist=["HEAD", "GET", "OPTIONS", "POST", "TRACE"])
        session.mount('https://', TimeoutHTTPAdapter(
            max_retries=retries,
            timeout=kwargs.get('timeout', DEFAULT_TIMEOUT)))
        # TODO: Figure out how to utilize this within the validate_response
        # TODO: This will be a much better way of handling bad requests than
        # TODO: what I'm currently doing.
        # session.hooks['response'] = \
        #     [lambda response, *args, **kwargs: response.raise_for_status()]
        session.headers.update({
            "User-Agent": random.choice(USER_AGENT_LIST)
        })
    return session 

def hit(self, request, title):
        params = {
            "v": "1",
            "tid": request.registry.settings['analytics'],
            "cid": "api",
            "t": "pageview",
            "dt": "API - {}".format(title),
            "dp": request.path
        }
        payload = urlencode(params)
        session = FuturesSession()
        debug_url = "/debug" if self.debug else ""
        r = session.get("https://www.google-analytics.com{}/collect?{}".format(debug_url, payload))
        if self.debug:
            response = r.result()
            print(response.content) 

def open(self):
        if self.request == 'hyper':
            if self.http2:
                self.__http = hyper.HTTP20Connection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers)
            else:
                self.__http = hyper.HTTPConnection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers)
        elif self.request == 'httpx':
            if self.http2:
                self.__http = httpx.AsyncClient(base_url='%s://%s' % (self.scheme, self.host), http2=self.http2)
            else:
                self.__http = httpx.Client(base_url='%s://%s' % (self.scheme, self.host))
        elif self.request == 'requests':
            self.__http = requests.Session()
            if self.using_proxy():
                self.__http.proxies = urllib.request.getproxies()
        elif self.request == 'requests-futures':
            self.__http = FuturesSession()
            if self.using_proxy():
                self.__http.proxies = urllib.request.getproxies()
        elif self.request == 'httplib2':
            self.__http = httplib2.Http()
        else:
            if self.scheme == 'http':
                self.__http = http_client.HTTPConnection(self.host, self.port)
            elif self.scheme == 'https':
                self.__http = http_client.HTTPSConnection(self.host, self.port)
                if self.using_proxy():
                    self.__http.set_tunnel(self.realhost, self.realport, self.proxy_headers) 

def send_requests_worker(load: list, delay: float, random_delay: bool, headers: dict):
    """
    Start sending out requests, given load (URLs), delay (delay to insert between requests) and headers (User-Agent
    for the requests)
    """
    if len(load) == 0:
        return 0, 0, 0  # return futs, timeouts, exceptions

    if sr_plugins.exists("circuit_breaker"):
        cb = sr_plugins.load("circuit_breaker")
        first_url = load[0]["url"]
        cb.main(url=first_url)

    session = FuturesSession(max_workers=5)

    if sr_plugins.exists("worker_headers"):
        worker_headers = sr_plugins.load("worker_headers")
    else:
        worker_headers = None

    futs = _send_futs_worker(
        session,
        load,
        delay=delay,
        random_delay=random_delay,
        headers=headers,
        worker_headers=worker_headers,
    )
    futs, timeouts, exceptions = _collect_futs_worker(futs)

    return futs, timeouts, exceptions 

def __init__(self, workers=4):
        # general session for sync api calls
        self.session = RedirectSession()
        self.session.headers.update({'User-Agent': _get_user_agent()})
        # ensure all calls to the session are throttled
        self.session.request = _Throttler().wrap(self.session.request)
        # the asyncpool is reserved for long-running async tasks
        self._asyncpool = FuturesSession(
            max_workers=workers,
            session=self.session) 

def get_lastfm_asyncrequest(method, **kwargs):
    ''' make an aysnc request using a requests_futures FuturesSession
        to the last.fm api
        return a Future() '''
    return SESSION.get(API_BASE, params={"method": method,
                                         **REQUEST_PARAMETERS,
                                         **kwargs}) 

def __get_attr_elements_async(self, limit=200000):
        """Get elements of report attributes asynchronously.
        Implements GET /reports/<report_id>/attributes/<attribute_id>/elements
        """

        attr_elements = []
        if self.attributes:
            threads = helper.get_parallel_number(len(self.attributes))
            with FuturesSession(executor=ThreadPoolExecutor(max_workers=threads),
                                session=self._connection.session) as session:
                # Fetch first chunk of attribute elements.
                futures = self.__fetch_attribute_elements_chunks(session, limit)
                pbar = tqdm(futures, desc="Loading attribute elements", leave=False, disable=(not self.progress_bar))
                for i, future in enumerate(pbar):
                    attr = self.attributes[i]
                    response = future.result()
                    if not response.ok:
                        helper.response_handler(response, "Error getting attribute " + attr['name'] + " elements")
                    elements = response.json()
                    # Get total number of rows from headers.
                    total = int(response.headers['x-mstr-total-count'])
                    for _offset in range(limit, total, limit):
                        response = reports.report_single_attribute_elements(connection=self._connection,
                                                                            report_id=self._report_id,
                                                                            attribute_id=attr["id"],
                                                                            offset=_offset,
                                                                            limit=limit,
                                                                            verbose=helper.debug())
                        elements.extend(response.json())
                    # Append attribute data to the list of attributes.
                    attr_elements.append({"attribute_name": attr['name'],
                                          "attribute_id": attr['id'],
                                          "elements": elements})
                pbar.close()

            return attr_elements 

def __get_attr_elements_async(self, limit=200000):
        """Get attribute elements. Implements GET /cubes/<cube_id>/attributes/<attribute_id>/elements"""

        attr_elements = []
        if self.attributes:
            threads = helper.get_parallel_number(len(self.attributes))
            with FuturesSession(executor=ThreadPoolExecutor(max_workers=threads),
                                session=self._connection.session) as session:
                # Fetch first chunk of attribute elements.
                futures = self.__fetch_attribute_elements_chunks(session, limit)
                pbar = tqdm(futures, desc="Loading attribute elements", leave=False, disable=(not self.progress_bar))
                for i, future in enumerate(pbar):
                    attr = self.attributes[i]
                    response = future.result()
                    if not response.ok:
                        helper.response_handler(response, "Error getting attribute " + attr["name"] + " elements")
                    elements = response.json()
                    # Get total number of rows from headers.
                    total = int(response.headers['x-mstr-total-count'])
                    for _offset in range(limit, total, limit):
                        response = cubes.cube_single_attribute_elements(connection=self._connection,
                                                                        cube_id=self._cube_id,
                                                                        attribute_id=attr["id"],
                                                                        offset=_offset,
                                                                        limit=limit,
                                                                        verbose=helper.debug())
                        elements.extend(response.json())
                    # Append attribute data to the list of attributes.
                    attr_elements.append({"attribute_name": attr['name'],
                                          "attribute_id": attr['id'],
                                          "elements": elements})
                pbar.close()

            return attr_elements 

def main():
    urls = {}
    requests = []
    session = FuturesSession(max_workers=10)
    for year in YEARS_TO_PARSE:
        landing_page = SEARCH_LANDING + '&year=' + str(year)
        landing_res = session.get(landing_page).result()
        landing_bs = BS(landing_res.content, 'html5lib')
        number_span = landing_bs.select('li.ep_tag_selected span')[0].text
        number_of_question = int(re.findall(r'\d+', number_span)[0])
        number_of_pages = math.ceil(
            number_of_question / 10)  # change to per page
        for page_num in range(1, number_of_pages + 1):
            res = session.get(landing_page + '&currentPage=' + str(page_num))
            requests.append(res)
        for request in tqdm(requests):
            try:
                request_result = request.result()
            except ConnectionError:
                print(
                    'Due to the ConnectionError page {} hasn\'t been parsed'.format(page_num))
                continue
            page = BS(request_result.content, "html5lib")
            if page:
                for notice in page.select('.results div.notice'):
                    for url in notice.select('ul.documents li a'):
                        title_text = notice.select(
                            'p.title a.result_details_link')[0].text
                        title_date = notice.select(
                            'div.date_reference span.date')[0].text
                        question_format = url.get('href').split('.')[-1]
                        title = '{} ({}).{}'.format(
                            title_text, title_date, question_format)
                        title = re.sub(r'[\n\r\t]', '', title)
                        title = title.replace('/', '-')
                        urls[url.get('href')] = title
            else:
                break
    if not os.path.exists(FOLDER_TO_DOWNLOAD):
        os.mkdir(FOLDER_TO_DOWNLOAD)
    download(urls, FOLDER_TO_DOWNLOAD) 

def generate_requests(hosts, jolokia_port, jolokia_prefix, jolokia_user, jolokia_password):
    """Return a generator of requests to fetch the under replicated
    partition number from the specified hosts.

    :param hosts: list of brokers ip addresses
    :type hosts: list of strings
    :param jolokia_port: HTTP port for Jolokia
    :type jolokia_port: integer
    :param jolokia_prefix: HTTP prefix on the server for the Jolokia queries
    :type jolokia_prefix: string
    :param jolokia_user: Username for Jolokia, if needed
    :type jolokia_user: string
    :param jolokia_password: Password for Jolokia, if needed
    :type jolokia_password: string
    :returns: generator of requests
    """
    session = FuturesSession()
    if jolokia_user and jolokia_password:
        session.auth = (jolokia_user, jolokia_password)
    for host in hosts:
        url = "http://{host}:{port}/{prefix}/read/{key}".format(
            host=host,
            port=jolokia_port,
            prefix=jolokia_prefix,
            key=UNDER_REPL_KEY,
        )
        yield host, session.get(url) 

def __init__(self, url, timeout=None, username="", password="",
                 loop=None, verify_ssl=False, future_class=None):
        super().__init__(url, timeout=timeout, username=username,
                         password=password, loop=loop,
                         validate_cert=verify_ssl, future_class=futures.Future)
        self._session = FuturesSession() 

def get_async():
    # [START requests_get]
    session = FuturesSession()
    url = 'http://www.google.com/humans.txt'

    rpc = session.get(url)

    # ... do other things ...

    resp = make_response(rpc.result().text)
    resp.headers['Content-type'] = 'text/plain'
    return resp
    # [END requests_get] 

def multi_futures_app_request(
    app_ids, headers=None, verify=True, params=None, workers=s.CONCURRENT_REQUESTS
):
    """
    :param app_ids: a list of app IDs.
    :param headers: a dictionary of custom headers to use.
    :param verify: bool for requesting SSL verification.
    :return: a list of all apps' detail data
    """
    session = FuturesSession(max_workers=workers)

    headers = default_headers() if headers is None else headers
    responses = [
        session.get(
            build_url("details", app_id),
            headers=headers,
            verify=verify,
            params=params,
            hooks={"response": parse_app_details_response_hook},
        )
        for app_id in app_ids
    ]

    apps = []
    for i, response in enumerate(responses):
        try:
            result = response.result()
            app_json = result.app_details_data
            app_json.update({"app_id": app_ids[i], "url": result.url})
            apps.append(response.result().app_details_data)
        except requests.exceptions.RequestException as e:
            log.error(
                "Error occurred fetching {app}: {err}".format(
                    app=app_ids[i], err=str(e)
                )
            )

    return apps 

def _get_data(self, key, params={}, **kwargs):
        config = self._CONFIG[key]
        params = self._construct_params(config, params)
        urls = self._construct_urls(config, params, **kwargs)
        response_field = config['response_field']
        try:
            if isinstance(self.session, FuturesSession):
                data = self._async_requests(
                    response_field, urls, params, **kwargs)
            else:
                data = self._sync_requests(
                    response_field, urls, params, **kwargs)
            return data
        except ValueError:
            return {'error': 'HTTP 404 Not Found.  Please try again'} 

def BinaryDataHTTPInstanceGenerator(action, instance_times, blocking_cli, data_file):
    """
    TODO: Automate content type
    """
    url = base_gust_url + action
    session = FuturesSession(max_workers=15)
    if len(instance_times) == 0:
        return False
    after_time, before_time = 0, 0

    try:
        data = binary_data_cache[data_file]
    except:
        data = open(data_file, 'rb').read()
        binary_data_cache[data_file] = data

    for t in instance_times:
        st = t - (after_time - before_time)
        if st > 0:
            time.sleep(st)
        before_time = time.time()
        session.post(url=url, headers={'Content-Type': 'image/jpeg'},
                     params={'blocking': blocking_cli, 'result': RESULT},
                     data=data, auth=(user_pass[0], user_pass[1]), verify=False)
        after_time = time.time()

    return True 

def HTTPInstanceGenerator(action, instance_times, blocking_cli, param_file=None):
    if len(instance_times) == 0:
        return False
    session = FuturesSession(max_workers=15)
    url = base_url + action
    parameters = {'blocking': blocking_cli, 'result': RESULT}
    authentication = (user_pass[0], user_pass[1])
    after_time, before_time = 0, 0

    if param_file == None:
        st = 0
        for t in instance_times:
            st = st + t - (after_time - before_time)
            before_time = time.time()
            if st > 0:
                time.sleep(st)
            future = session.post(url, params=parameters, auth=authentication, verify=False)
            after_time = time.time()
    else:   # if a parameter file is provided
        try:
            param_file_body = param_file_cache[param_file]
        except:
            with open(param_file, 'r') as f:
                param_file_body = json.load(f)
                param_file_cache[param_file] = param_file_body

        for t in instance_times:
            st = t - (after_time - before_time)
            if st > 0:
                time.sleep(st)
            before_time = time.time()
            future = session.post(url, params=parameters, auth=authentication,
                                  json=param_file_body, verify=False)
            after_time = time.time()

    return True 

def __init__(self, host='localhost', port=8094, tags=None):
        # only import HttpClient's dependencies if using HttpClient
        # if they're not found, inform the user how to install them
        try:
            from requests_futures.sessions import FuturesSession
        except ImportError:
            raise ImportError('pytelegraf[http] must be installed to use HTTP transport')

        super(HttpClient, self).__init__(host, port, tags)

        # the default url path for writing metrics to Telegraf is /write
        self.url = 'http://{host}:{port}/write'.format(host=self.host, port=self.port)

        # create a session to reuse the TCP socket when possible
        self.future_session = FuturesSession() 

def __init__(
        self, default_headers=None, max_requests=10, rate_limit=0,
        req_timeout=None, max_retry=10, total_retry=100, drop_404s=False,
    ):
        """Create the MultiRequest.

        Args:
            default_headers - A dict of headers which will be added to every request
            max_requests - Maximum number of requests to issue at once
            rate_limit - Maximum number of requests to issue per second
            req_timeout - Maximum number of seconds to wait without reading a response byte before deciding an error has occurred
            max_retry - The total number of attempts to retry a single batch of requests
            total_retry - The total number of request retries that can be made through the entire session
        Note there is a difference between `max_retry` and `total_retry`:
            - `max_retry` refers to how many times a batch of requests will be re-issued collectively
            - `total_retry` refers to a limit on the total number of outstanding requests made
            Once the latter is exhausted, no failed request within the whole session will be retried.
        """
        self._default_headers = default_headers
        self._max_requests = max_requests
        self._req_timeout = req_timeout or 25.0
        self._max_retry = max_retry
        self._drop_404s = drop_404s
        self._rate_limiter = RateLimiter(rate_limit) if rate_limit else None
        self._availability_limiter = AvailabilityLimiter(total_retry) if total_retry else None
        self._session = FuturesSession(max_workers=max_requests)
        retries = Retry(total=0, status_forcelist=[500, 502, 503, 504], raise_on_status=True)
        self._session.mount(
            'https://', SSLAdapter(
                max_retries=retries, pool_maxsize=max_requests, pool_connections=max_requests,
            ),
        ) 

def get_callback():
    global response_text
    global counter

    response_text = ''
    counter = 0

    def cb(resp, *args, **kwargs):
        global response_text
        global counter

        if 300 <= resp.status_code < 400:
            return  # ignore intermediate redirection responses

        counter += 1
        response_text += 'Response number {} is {} bytes from {}\n'.format(
            counter, len(resp.text), resp.url)

    session = FuturesSession()
    urls = [
        'https://google.com/',
        'https://www.google.com/humans.txt',
        'https://www.github.com',
        'https://www.travis-ci.org'
    ]

    futures = [session.get(url, hooks={'response': cb}) for url in urls]

    # No wait functionality in requests_futures, so check every second to
    # see if all callbacks are done, up to TIMEOUT seconds
    for elapsed_time in range(TIMEOUT+1):
        all_done = True
        for future in futures:
            if not future.done():
                all_done = False
                break
        if all_done:
            break
        sleep(1)

    resp = make_response(response_text)
    resp.headers['Content-type'] = 'text/plain'
    return resp 

def __init__(self, **kwargs):
        self.logger = logging.getLogger('gbdxtools')
        self.logger.setLevel(logging.ERROR)
        self.console_handler = logging.StreamHandler()
        self.console_handler.setLevel(logging.ERROR)
        self.formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        self.console_handler.setFormatter(self.formatter)
        self.logger.addHandler(self.console_handler)
        self.logger.info('Logger initialized')

        if 'host' in kwargs:
            self.root_url = 'https://%s' % kwargs.get('host')

        if (kwargs.get('username') and kwargs.get('password')):
            self.gbdx_connection = gbdx_auth.session_from_kwargs(**kwargs)
        elif kwargs.get('gbdx_connection'):
            self.gbdx_connection = kwargs.get('gbdx_connection')
        elif self.gbdx_connection is None:
            # This will throw an exception if your .ini file is not set properly
            self.gbdx_connection = gbdx_auth.get_session(kwargs.get('config_file'))

        def expire_token(r, *args, **kw):
            """
            Requests a new token if 401, retries request, mainly for auth v2 migration
            :param r:
            :param args:
            :param kw:
            :return:
            """
            if r.status_code == 401:
                try:
                    # remove hooks so it doesn't get into infinite loop
                    r.request.hooks = None
                    # expire the token
                    gbdx_auth.expire_token(token_to_expire=self.gbdx_connection.token,
                                           config_file=kwargs.get('config_file'))
                    # re-init the session
                    self.gbdx_connection = gbdx_auth.get_session(kwargs.get('config_file'))
                    # make original request, triggers new token request first
                    return self.gbdx_connection.request(method=r.request.method, url=r.request.url)

                except Exception as e:
                    r.request.hooks = None
                    print("Error expiring token from session, Reason {}".format(e))

        if self.gbdx_connection is not None:

            self.gbdx_connection.hooks['response'].append(expire_token)

            # status_forcelist=[500, 502, 504]))
            self.gbdx_connection.mount(VIRTUAL_RDA_URL, HTTPAdapter(max_retries=5))

        self.gbdx_futures_session = FuturesSession(session=self.gbdx_connection, max_workers=64)

        if 'GBDX_USER' in os.environ:
            header = {'User-Agent': os.environ['GBDX_USER']}
            self.gbdx_futures_session.headers.update(header)
            self.gbdx_connection.headers.update(header) 

def get_subjurisdictions(self):
        """
        Returns a list of subjurisdictions depending on the level
        of the main jurisdiction. States always have counties, and
        counties and cities may have precincts.
        """

        subjurisdictions_url = self._get_subjurisdictions_url()
        if 'Web02' in self.url:
            json_url = self.get_latest_summary_url(self.url).replace('summary.json', 'electionsettings.json')
            try:
                r = requests.get(json_url)
                r.raise_for_status()
                jurisdictions = []
                counties = r.json()['settings']['electiondetails']['participatingcounties']
                jurisdictions = self._get_subjurisdictions_urls_from_json(counties)
                return jurisdictions
            except requests.exceptions.HTTPError:
                return []
        elif not subjurisdictions_url:
            json_url = self.url.replace('summary.html', 'json/electionsettings.json')
            try:
                r = requests.get(json_url)
                r.raise_for_status()
                jurisdictions = []
                counties = r.json()['settings']['electiondetails']['participatingcounties']
                jurisdictions = self._get_subjurisdictions_urls_from_json(counties)
                return jurisdictions
            except requests.exceptions.HTTPError:
                json_url = self.url.replace('summary.html', 'json/en/electionsettings.json')
                try:
                    r = requests.get(json_url)
                    r.raise_for_status()
                    jurisdictions = []
                    counties = r.json()['settings']['electiondetails']['participatingcounties']
                    jurisdictions = self._get_subjurisdictions_urls_from_json(counties)
                    return jurisdictions
                except requests.exceptions.HTTPError:
                    return []
        try:
            r = requests.get(subjurisdictions_url)
            r.raise_for_status()

            # Use a maximum of 10 workers.  Should we parameterize this?
            session = FuturesSession(max_workers=10)
            future_to_name = {}
            for url, name in self._scrape_subjurisdiction_paths(r.text):
                future = self._subjurisdiction_url_future(session, url)
                future_to_name[future] = name

            jurisdictions = []
            for future in concurrent.futures.as_completed(future_to_name):
                url = self._subjurisdiction_url_from_future(future)
                name = future_to_name[future]
                jurisdictions.append(Jurisdiction(url, 'county', name))

            return jurisdictions
        except requests.exceptions.HTTPError:
            return []