Python six.moves.urllib.request() Examples

def call(self, url, **params):
        url = six.moves.urllib.parse.urljoin(self.base_url, url)
        if self.verbose:
            log.info("Import API URL: %s", url)

        params = self.sign(six.moves.urllib.parse.urlparse(url).path, list(params.items()))

        while True:
            try:
                result = six.moves.urllib.request.urlopen(url, six.moves.urllib.parse.urlencode(params))
                resp = result.read()
                return json.loads(resp)
            except six.moves.urllib.error.HTTPError as e:
                e.msg += ' ({0})'.format(url)
                if self.verbose:
                    error_content = e.read()
                    e.msg += '. Error response:\n' + error_content
                raise e
            except (six.moves.urllib.error.URLError, IOError):
                if self.retry:
                    log.exception('Error making API request, will retry')
                    continue
                raise 

def test_check_phone_verification_ok(self, phone_service):
        with h.push_config(config, **{'project.verify_phone': 'true'}):
            phone_service.check.return_value = {'status': 'ok'}
            req_id = 'request-id'

            # make request to verify first to initialize session
            phone_service.verify.return_value = {
                'request_id': req_id, 'status': 'ok'}
            r = self.app.get('/p/verify_phone', {'number': '11234567890'})

            r = self.app.get('/p/check_phone_verification', {'pin': '1234'})
            assert_equal(r.json, {'status': 'ok'})
            phone_service.check.assert_called_once_with(req_id, '1234')

            user = M.User.by_username('test-admin')
            hash = user.get_tool_data('phone_verification', 'number_hash')
            assert_equal(hash, '54c61c96d5d5aea5254c2d4f41508a938e5501b4') 

def test_check_phone_verification_error(self, phone_service):
        with h.push_config(config, **{'project.verify_phone': 'true'}):
            phone_service.check.return_value = {'status': 'error'}
            req_id = 'request-id'

            # make request to verify first to initialize session
            phone_service.verify.return_value = {
                'request_id': req_id, 'status': 'ok'}
            r = self.app.get('/p/verify_phone', {'number': '1234567890'})

            r = self.app.get('/p/check_phone_verification', {'pin': '1234'})
            assert_equal(r.json, {'status': 'error'})
            phone_service.check.assert_called_once_with(req_id, '1234')

            user = M.User.by_username('test-admin')
            hash = user.get_tool_data('phone_verification', 'number_hash')
            assert_equal(hash, None) 

def _request(self, arguments, timeout=None):
        """Assembles the url and performs a get request to
        the MicroStrategy Task Service API

        Args:
            arguments (dict): Maps get key parameters to values

        Returns:
            str: the xml text response
        """

        arguments.update(BASE_PARAMS)
        request = self._base_url + six.moves.urllib.parse.urlencode(arguments)
        logger.info("submitting request %s" % request)

        try:
            response = requests.get(request, timeout=timeout)
        except requests.exceptions.Timeout as e:
            raise MstrClientException(str(e))

        if not response.ok:
            raise MstrClientException(response.text)

        logger.info("received response %s" % response.text)
        return response.text 

def fetch_bibtex_by_fulltext_scholar(txt, assess_results=True):
    import scholarly.scholarly
    scholarly._get_page = _get_page_fast  # remove waiting time
    logger.debug(txt)
    search_query = scholarly.search_pubs_query(txt)

    # get the most likely match of the first results
    results = list(search_query)
    if len(results) > 1 and assess_results:
        maxscore = 0
        result = results[0]
        for res in results:
            score = _scholar_score(txt, res.bib)
            if score > maxscore:
                maxscore = score
                result = res
    else:
        result = results[0]

    # use url_scholarbib to get bibtex from google
    if getattr(result, 'url_scholarbib', ''):
        bibtex = scholarly._get_page(result.url_scholarbib).strip()
    else:
        raise NotImplementedError('no bibtex import linke. Make crossref request using title?')
    return bibtex 

def test_heal_vnf_instance_already_not_instantiated(self,
            mock_log, mock_get_lock):
        vnf_package_vnfd = self._create_and_upload_vnf_package()
        vnf_instance_data = fake_obj.get_vnf_instance_data(
            vnf_package_vnfd.vnfd_id)

        vnf_instance_data['instantiation_state'] =\
            fields.VnfInstanceState.NOT_INSTANTIATED
        vnf_instance = objects.VnfInstance(context=self.context,
                                           **vnf_instance_data)
        vnf_instance.create()

        heal_vnf_req = objects.HealVnfRequest(cause="healing request")
        self.conductor.heal(self.context, vnf_instance, heal_vnf_req)

        self.vnflcm_driver.heal_vnf.assert_not_called()
        expected_log = ('Heal action cannot be performed on vnf %(id)s '
                        'which is in %(state)s state.')
        mock_log.error.assert_called_once_with(expected_log,
            {'id': vnf_instance.id,
             'state': fields.VnfInstanceState.NOT_INSTANTIATED}) 

def prefetch_metrics(self, container_id):
        """Initiates requesting invoking `stats` for the specified container.  If you invoke this, you must
        also eventually invoke `get_metrics` with the same container.  By invoking this first, the `get_metrics`
        call will take less time when issuing many `stats` requests.

        Whenever possible, you should first invoke this method for all containers whose metrics you wish to request
        before any call to `get_metrics`.

        The behavior is not well defined if you invoke `prefetch_metrics` multiple times for a container before
        invoking `get_metrics` for it.

        @param container_id: The id of the container to fetch.
        @type container_id: str
        """
        self.__lock.acquire()
        try:
            if container_id not in self.__container_scoreboard:
                self._add_fetch_task(container_id)
        finally:
            self.__lock.release() 

def __start_workers(self, count):
        """Start `count` worker threads that will fetch metrics results.

        @param count:  The number of threads to start.
        @type count: int
        """
        new_number_workers = min(self.__concurrency, count + self.__num_worker_threads)
        for i in range(self.__num_worker_threads, new_number_workers):
            x = threading.Thread(target=self.__worker)
            # Set daemon so this thread does not need to be finished for the overall process to stop.  This allows
            # the process to terminate even if a `stats` request is still in-flight.
            x.setDaemon(True)
            x.start()
            self.__num_worker_threads += 1
            # For accounting purposes,we consider the thread idle until it actually has a container it is fetching.
            self.__idle_workers_count += 1 

def setUp(self):
        super(XMLAPIConnectorTest, self).setUp()

        emc_share_driver = fakes.FakeEMCShareDriver()

        self.configuration = emc_share_driver.configuration

        xml_socket = mock.Mock()
        xml_socket.read = mock.Mock(return_value=XML_CONN_TD.FAKE_RESP)
        opener = mock.Mock()
        opener.open = mock.Mock(return_value=xml_socket)

        with mock.patch.object(url_request, 'build_opener',
                               mock.Mock(return_value=opener)):
            self.XmlConnector = connector.XMLAPIConnector(
                configuration=self.configuration, debug=False)

            expected_calls = [
                mock.call(XML_CONN_TD.req_auth_url(),
                          XML_CONN_TD.req_credential(),
                          XML_CONN_TD.req_url_encode()),
            ]

            url_request.Request.assert_has_calls(expected_calls) 

def __init__(self, url):
    self._url = url
    self._opener = urllib.request.build_opener(
        CustomHTTPErrorHandler(),
        urllib.request.HTTPCookieProcessor(cookielib.CookieJar()))
    self._xsrf_token = None 

def request(self, resource, body=None, headers=None, method=None):
    """Sends HTTP request."""
    if not resource.startswith(self._url):
      assert resource.startswith('/')
      resource = self._url + resource
    req = urllib.request.Request(resource, body, headers=(headers or {}))
    if method:
      req.get_method = lambda: method
    resp = self._opener.open(req)
    return self.HttpResponse(resp.getcode(), resp.read(), resp.info()) 

def login_as_admin(self, user='test@example.com'):
    """Performs dev_appserver login as admin, modifies cookies."""
    self.request('/_ah/login?email=%s&admin=True&action=Login' % user)
    self._xsrf_token = None 

def request(self, resource, body=None, headers=None, method=None):
    """Sends HTTP request."""
    if not resource.startswith(self._url):
      assert resource.startswith('/')
      resource = self._url + resource
    req = urllib.request.Request(resource, body, headers=(headers or {}))
    if method:
      req.get_method = lambda: method
    resp = self._opener.open(req)
    return self.HttpResponse(resp.getcode(), resp.read(), resp.info()) 

def json_request(self, resource, body=None, headers=None, method=None):
    """Sends HTTP request and returns deserialized JSON."""
    if body is not None:
      body = json.dumps(body)
      headers = (headers or {}).copy()
      headers['Content-Type'] = 'application/json; charset=UTF-8'
    resp = self.request(resource, body, headers=headers, method=method)
    try:
      value = json.loads(resp.body)
    except ValueError:
      raise ValueError('Invalid JSON: %r' % resp.body)
    return self.HttpResponse(resp.http_code, value, resp.headers) 

def test_heal_vnf_instance(self, mock_get_lock):
        vnf_package_vnfd = self._create_and_upload_vnf_package()
        vnf_instance_data = fake_obj.get_vnf_instance_data(
            vnf_package_vnfd.vnfd_id)
        vnf_instance = objects.VnfInstance(context=self.context,
                                           **vnf_instance_data)
        vnf_instance.create()
        vnf_instance.instantiation_state = \
            fields.VnfInstanceState.INSTANTIATED
        vnf_instance.save()
        heal_vnf_req = objects.HealVnfRequest(cause="healing request")
        self.conductor.heal(self.context, vnf_instance, heal_vnf_req)
        self.vnflcm_driver.heal_vnf.assert_called_once_with(
            self.context, mock.ANY, heal_vnf_req) 

def test_fetch_vnf_package_content_valid_range(self):
        request = fake_request.HTTPRequest.blank(
            '/vnf_packages/%s/package_content/')
        request.headers["Range"] = 'bytes=10-99'
        range_ = self.controller._get_range_from_request(request, 120)
        self.assertEqual(10, range_.start)
        self.assertEqual(100, range_.end)  # non-inclusive 

def test_fetch_vnf_package_content_invalid_range(self):
        request = fake_request.HTTPRequest.blank(
            '/vnf_packages/%s/package_content/')
        request.headers["Range"] = 'bytes=150-'
        self.assertRaises(exc.HTTPRequestRangeNotSatisfiable,
                          self.controller._get_range_from_request,
                          request, 120) 

def test_fetch_vnf_package_content_invalid_multiple_range(self):
        request = fake_request.HTTPRequest.blank(
            '/vnf_packages/%s/package_content/')
        request.headers["Range"] = 'bytes=10-20,21-30'
        self.assertRaises(exc.HTTPBadRequest,
                          self.controller._get_range_from_request, request,
                          120) 

def test_enforce_http_false(self):

        def fakeurlopen(url, post_data):
            return six.StringIO("False")

        with mock.patch.object(urlrequest, 'urlopen', new=fakeurlopen):
            action = "example:get_http"
            target = {}
            self.assertRaises(exceptions.PolicyNotAuthorized, policy.enforce,
                              self.context, action, target) 

def testUnspecifiedContentEncoding(self):
        data = 'regular non-gzipped content'
        with patch.object(urllib_request, 'urlopen',
                          return_value=MockRequestResponse(data, '')):
            self.assertEqual(data, util._GetURLContent('unused_url_parameter')) 

def testGZippedContent(self):
        data = u'¿Hola qué tal?'
        compressed_data = _Gzip(data.encode('utf-8'))
        with patch.object(urllib_request, 'urlopen',
                          return_value=MockRequestResponse(
                              compressed_data, 'gzip')):
            self.assertEqual(data, util._GetURLContent(
                'unused_url_parameter').decode('utf-8')) 

def test_verify_phone(self, phone_service):
        with h.push_config(config, **{'project.verify_phone': 'true'}):
            phone_service.verify.return_value = {
                'request_id': 'request-id', 'status': 'ok'}
            r = self.app.get('/p/verify_phone', {'number': '1-555-444-3333'})
            phone_service.verify.assert_called_once_with('15554443333')
            assert_equal(r.json, {'status': 'ok'})
            rid = r.session.get('phone_verification.request_id')
            hash = r.session.get('phone_verification.number_hash')
            assert_equal(rid, 'request-id')
            assert_equal(hash, 'f9ac49faef45d18746ced08d001e23b179107940') 

def test_verify_phone_already_used(self, phone_service):
        with h.push_config(config, **{'project.verify_phone': 'true'}):
            u = M.User.register(dict(username='existing-user'), make_project=False)
            u.set_tool_data('phone_verification', number_hash=utils.phone_number_hash('1-555-444-9999'))
            session(u).flush(u)
            phone_service.verify.return_value = {'request_id': 'request-id', 'status': 'ok'}
            r = self.app.get('/p/verify_phone', {'number': '1-555-444-9999'})
            assert_equal(r.json, {
                'status': 'error',
                'error': 'That phone number has already been used.'
            }) 

def setUp(self):
    super(HbaseTest, self).setUp()
    FLAGS['hbase_version'].parse('1.3.2.1')
    p = mock.patch.object(urllib.request, 'urlopen')
    self.mock_url_open = p.start()
    self.addCleanup(p.stop) 

def json_request(self, resource, body=None, headers=None, method=None):
    """Sends HTTP request and returns deserialized JSON."""
    if body is not None:
      body = json.dumps(body)
      headers = (headers or {}).copy()
      headers['Content-Type'] = 'application/json; charset=UTF-8'
    resp = self.request(resource, body, headers=headers, method=method)
    try:
      value = json.loads(resp.body)
    except ValueError:
      raise ValueError('Invalid JSON: %r' % resp.body)
    return self.HttpResponse(resp.http_code, value, resp.headers) 

def calculate_token(self, text, seed=None):
		""" Calculate the request token (`tk`) of a string
		:param text: str The text to calculate a token for
		:param seed: str The seed to use. By default this is the number of hours since epoch
		"""

		if seed is None:
			seed = self._get_token_key()

		[first_seed, second_seed] = seed.split(".")

		try:
			d = bytearray(text.encode('UTF-8'))
		except UnicodeDecodeError:
			# This will probably only occur when d is actually a str containing UTF-8 chars, which means we don't need
			# to encode.
			d = bytearray(text)

		a = int(first_seed)
		for value in d:
			a += value
			a = self._work_token(a, self.SALT_1)
		a = self._work_token(a, self.SALT_2)
		a ^= int(second_seed)
		if 0 > a:
			a = (a & 2147483647) + 2147483648
		a %= 1E6
		a = int(a)
		return str(a) + "." + str(a ^ int(first_seed)) 

def save(self, savefile):
		""" Do the Web request and save to `savefile` """
		with open(savefile, 'wb') as f:
			self.write_to_fp(f) 

def write_to_fp(self, fp):
		""" Do the Web request and save to a file-like object """
		for idx, part in enumerate(self.text_parts):
			payload = { 'ie' : 'UTF-8',
						'q' : part,
						'tl' : self.lang,
						'ttsspeed' : self.speed,
						'total' : len(self.text_parts),
						'idx' : idx,
						'client' : 'tw-ob',
						'textlen' : self._len(part),
						'tk' : self.token.calculate_token(part)}
			headers = {
				"Referer" : "http://translate.google.com/",
				"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36"
			}
			if self.debug: print(payload)
			try:
				# Disable requests' ssl verify to accomodate certain proxies and firewalls
				# Filter out urllib3's insecure warnings. We can live without ssl verify here
				with warnings.catch_warnings():
					warnings.filterwarnings("ignore", category=requests.packages.urllib3.exceptions.InsecureRequestWarning)
					r = requests.get(self.GOOGLE_TTS_URL,
									 params=payload,
									 headers=headers,
									 proxies=urllib.request.getproxies(),
									 verify=False)
				if self.debug:
					print("Headers: {}".format(r.request.headers))
					print("Request url: {}".format(r.request.url))
					print("Response: {}, Redirects: {}".format(r.status_code, r.history))
				r.raise_for_status()
				for chunk in r.iter_content(chunk_size=1024):
					fp.write(chunk)
			except Exception as e:
				raise 

def logRequest(self, body, timeout=None, _async=True):
        # Log the real request method
        method = self.method
        if not method:
            method = body is not None and "POST" or "GET"
        util.LOG(
            "Starting request: {0} {1} (async={2} timeout={3})".format(method, util.cleanToken(self.url),
                                                                       _async, timeout)
        ) 

def _get_cookie_crumb():
    '''
    This function perform a query and extract the matching cookie and crumb.
    '''
    global cookier, _cookie, _crumb

    # Perform a Yahoo financial lookup on SP500
    cookier.cookiejar.clear()
    req = urllib.request.Request(
        'https://finance.yahoo.com/quote/^GSPC', headers=_headers)
    f = urllib.request.urlopen(req, timeout=5)
    alines = f.read().decode('utf-8')

    # Extract the crumb from the response
    cs = alines.find('CrumbStore')
    cr = alines.find('crumb', cs + 10)
    cl = alines.find(':', cr + 5)
    q1 = alines.find('"', cl + 1)
    q2 = alines.find('"', q1 + 1)
    crumb = alines[q1 + 1:q2]
    _crumb = crumb

    # Extract the cookie from cookiejar
    for c in cookier.cookiejar:
        if c.domain != '.yahoo.com':
            continue
        if c.name != 'B':
            continue
        _cookie = c.value

    # Print the cookie and crumb
    #print('Cookie:', _cookie)
    #print('Crumb:', _crumb)