Python grequests.map() Examples

def get_image_urls(girl_urls):

    girl_list = []

    # 建立5个并发连接
    rs = (grequests.get(url) for url in girl_urls)
    responses = grequests.map(rs, size = 5)

    for response in responses:
        parsed_body = html.fromstring(response.text)
        girl_title  = parsed_body.xpath('//title/text()')
        image_urls = parsed_body.xpath('//li[@class="slide "]/img/@src | //li[@class="slide "]/img/@delay')

        # print image_urls
        girl_dict = {girl_title[0] : image_urls}
        girl_list.append(girl_dict)
    
    print "get_girl_urls done!!!"
    return girl_list 

def _get_async(self, keys, frmt='json', params={}):
        # does not work under pyhon3 so local import
        import grequests
        session = self._get_session()
        try:
            # build the requests
            urls = self._get_all_urls(keys, frmt)
            self.logging.debug("grequests.get processing")
            rs = (grequests.get(url, session=session, params=params)  for key,url in zip(keys, urls))
            # execute them
            self.logging.debug("grequests.map call")
            ret = grequests.map(rs, size=min(self.settings.CONCURRENT, len(keys)))
            self.last_response = ret
            self.logging.debug("grequests.map call done")
            return ret
        except Exception as err:
            self.logging.warning("Error caught in async. " + err.message)
            return [] 

def get_responses(payloads):
        """

        Args:
          payloads(list): list of payloads for GET request

        Returns:
          list: list of dictionaries with data from JSON responses

        """
        batch_url = "{base}stock/market/batch?".format(base=iex_url_base)
        rs = (grequests.get(batch_url, params=payload) for payload in payloads)
        result = grequests.map(rs)
        try:
            outputs = [r.json() for r in result]
        except AttributeError:
            outputs = []
        return outputs 

def get_volume(self):
        """

        Args:

        Returns:
          pandas.DataFrame: pandas.DataFrame with volumes of symbols

        """
        dfs = map(self.iex_get_volume, [[batch] for batch in self.batches])
        self.volume = pd.concat(dfs)
        # with Pool() as pool:
        #     self.volume = pd.concat(
        #         pool.starmap(self.iex_get_volume, [
        #                      [batch] for batch in self.batches])
        #     ) 

def filter_projects_by_license(projects, headers, licenses):
    import grequests

    reqs = [grequests.get(constants.REPO_URL.format(full_name=p.full_name), headers=headers)
            for p in projects]
    resps = grequests.map(reqs, exception_handler=request_exception_handler)

    filtered_projects = []
    for i, project in enumerate(projects):
        resp = resps[i]
        if not resp or resp.status_code != 200:
            logging.warning("ignoring %s because no info could be fetched", project.full_name)
            continue

        project_license = resp.json().get("license")
        if not project_license or not project_license.get("spdx_id"):
            continue
        license_id = project_license.get("spdx_id")
        if license_id in licenses:
            project.license = license_id
            filtered_projects.append(project)
    return filtered_projects 

def exploitpoc(self):
        for poctype in [
            'cms',
            'system',
            'industrial',
            'information',
                'hardware']:
            tasks = [
                grequests.post(
                    "http://tools.hexlt.org/api/" +
                    poctype,
                    json={
                        "url": self.url,
                        "type": type}) for type in self.poclist[poctype]]
            res = grequests.map(tasks, size=30)
            for i in res:
                result = i.json()
                if result['status']:
                    self.result.append(result['pocresult'])
        return self.result  # exp利用成功以列表形式返回结果,否则返回空列表 [] 

def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=5, auth=None, post_data=None):
    """
    @param post_data: If not None, do a POST request, with the passed data (which should be in the correct string format already)
    """
    headers = {'Connection': 'close', }  # no keepalive
    if auth:
        # auth should be a (username, password) tuple, if specified
        headers['Authorization'] = http_basic_auth_str(auth[0], auth[1])

    try:
        if post_data is not None:
            if is_json:
                headers['content-type'] = 'application/json'
            r = grequests.map((grequests.post(url, data=post_data, timeout=fetch_timeout, headers=headers, verify=False),))[0]
        else:
            r = grequests.map((grequests.get(url, timeout=fetch_timeout, headers=headers, verify=False),))[0]
        if r is None:
            raise Exception("result is None")
    except Exception as e:
        raise Exception("Got get_url request error: %s" % e)
    else:
        if r.status_code != 200 and abort_on_error:
            raise Exception("Bad status code returned: '%s'. result body: '%s'." % (r.status_code, r.text))
    return r.json() if r.text and is_json else r.text 

def test_015_test_fan(self):
        """ Test getting fan speed in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/fan/rack_1/40000000/0042'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_020_test_scan(self):
        """ Test scanning a board in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/scan/rack_1/40000000'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_015_test_led(self):
        """ Test setting led state (off) in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/led/rack_1/40000000/0300/off'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_014_test_led(self):
        """ Test setting led state (on) in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/led/rack_1/40000000/0300/on'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_013_test_led(self):
        """ Test getting led state in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/led/rack_1/40000000/0300'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_012_test_location(self):
        """ Test getting location info in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/location/rack_1/40000000'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_010_test_boot_target(self):
        """ Test setting boot target info in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/boot_target/rack_1/40000000/0200/pxe'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_009_test_boot_target(self):
        """ Test getting boot target info in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/boot_target/rack_1/40000000/0200'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_008_test_asset_info(self):
        """ Test getting asset info in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/asset/rack_1/40000000/0200'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_007_test_power(self):
        """ Test power control (on) in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/power/rack_1/40000000/0100/on'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_006_test_power(self):
        """ Test power control (off) in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/power/rack_1/40000000/0100/off'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_004_test_read(self):
        """ Test reading a temperature sensor in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/read/temperature/rack_1/40000000/0011'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_017_test_scan_all(self):
        """ Test issuing a scan all command in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/scan'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_018_test_scan_all(self):
        """ Test issuing a force scan all command in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/scan/force'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_019_test_scan(self):
        """ Test scanning a rack in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/scan/rack_1'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_001_test(self):
        """ Test hitting the 'test' endpoint of OpenDCRE in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/test'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def get(self, url, params={}, headers=None, format_='json'):
        """Calls the get method for a REST endpoint. 

        Args:
            url (str): URL of endpoint
            params (url): params for endpoint
            headers (dict, optional): any additional header attrs. Default to None
            format_ (str, optional): Format requested. Default to json

        Returns:
            dict: Response of request if format is json
            str: Response of request if format is not json
        """
        try:
            greq = grequests.get(url, params=params, headers=headers)
            response = grequests.map([greq])[0]
        except requests.exceptions.ConnectionError:
            print 'ConnectionError'
            return None
        self.info(response.url, _format=False)

        if format_ == 'json':
            try:
                json_data = response.json()
            except ValueError:
                json_data = None
                self.warn('Could not decode json properly')
            return json_data
        else:
            return response.text 

def get_events_for_season(self, season):
        responses = []
        season_events = []
        season_events_url = self.seasons_url+"/{0}/events".format(str(season))
        season_events_json = self.get_json(season_events_url)
        if season_events_json:
            rs = (grequests.get(event['$ref']) for event in season_events_json['items'])
            responses = grequests.map(rs)
            for response in responses:
                event_json = response.json()
                venue_json = self.get_json(event_json['venues'][0]['$ref'])
                season_events.append({"url": event_json['$ref'], "match_id": event_json['id'], "class": event_json['competitions'][0]['class']['generalClassCard'], "date": event_json['date'], "description": event_json['shortDescription'], "venue_url": event_json['venues'][0]['$ref'], "venue": venue_json['fullName']})
            return season_events
        else:
            return None 

def getpost(uid,query_urls):
    url='http://%s.tumblr.com/api/read?&num=50'%uid
    page=grequests.map([grequests.get(url)])[0].content
    total=re.findall('<posts start="0" total="(.*?)">',page)[0]
    total=int(total)
    a=[i*50 for i in range(total/50)]
    ul=api_url%uid
    for i in a:
        query_url=ul+str(i)
        query_urls.append(query_url) 

def run(query_urls):
    rs=[grequests.get(url) for url in query_urls]
    responses=grequests.map(rs,size=10)
    for resp in responses:
        content = resp.content
        videos = extractvideore.findall(content)
        video_links.extend([(v[0],vhead % v[1]) for v in videos])
        pic_links.extend(extractpicre.findall(content)) 

def run(query_urls):
    rs=[grequests.get(url) for url in query_urls]
    responses=grequests.map(rs,size=10)
    for resp in responses:
        content = resp.content
        videos = extractvideore.findall(content)
        video_links.extend([(v[0],vhead % v[1]) for v in videos])
        pic_links.extend(extractpicre.findall(content)) 

def load_thumbs(self):
        reqs = []
        providers = []
        for _ in range(20):
            base_url, provider = url_queue.get()
            providers.append(provider)
            proxied_url = f'{PROXY_URL}{base_url}'
            reqs.append(grequests.get(proxied_url))
        thumb_responses = grequests.map(reqs)
        record_stats(thumb_responses, providers)
        print_current_stats() 

def make_batch_request(requests):
    ''' Maps each given request to a response, and returns a list of the
    responses.

    :param requests: list of request objects
    :return: list of response objects
    '''
    responses = []
    responses.extend(grequests.map(requests))
    return responses