Python six.moves.urllib.request.Request() Examples

def authenticate(self, url, params):
        """
        returns a request object ready to be issued to the Mixpanel API
        """
        params['api_key'] = self.client.api_key
        params['expire'] = int(time.time()) + self.DEFAULT_EXPIRATION

        # Creating signature
        if 'sig' in params:
            del params['sig']
        params['sig'] = self._hash_args(params, self.client.api_secret)

        request_url = '{base_url}?{encoded_params}'.format(
            base_url=url,
            encoded_params=_unicode_urlencode(params)
        )
        return url_request.Request(request_url) 

def invoke_storlet_on_copy_dest(self):
        # No COPY in swiftclient. Using urllib instead...
        url = '%s/%s/%s' % (self.url, self.container, self.storlet_file)
        headers = {'X-Auth-Token': self.token,
                   'X-Run-Storlet': self.storlet_name,
                   'X-Object-Meta-Name': 'thumbnail',
                   'Destination': '%s/gen_thumb_on_copy_.jpg' % self.container}
        headers.update(self.additional_headers)
        req = Request(url, headers=headers)
        req.get_method = lambda: 'COPY'
        conn = urlopen(req, timeout=10)
        status = conn.getcode()
        self.assertIn(status, [201, 202])

        headers = c.head_object(self.url, self.token,
                                self.container, 'gen_thumb_on_copy_.jpg')
        self.assertLess(int(headers['content-length']), 1087318)
        self.assertEqual('thumbnail', headers['x-object-meta-name'])
        self.assertTrue('x-object-meta-x-timestamp' not in headers)
        self.assertTrue('x-timestamp' in headers) 

def fetch_saml_validation(self, ticket):
        # We do the SAML validation
        headers = {
            'soapaction': 'http://www.oasis-open.org/committees/security',
            'cache-control': 'no-cache',
            'pragma': 'no-cache',
            'accept': 'text/xml',
            'connection': 'keep-alive',
            'content-type': 'text/xml; charset=utf-8',
        }
        params = [('TARGET', self.service_url)]
        saml_validate_url = urllib_parse.urljoin(
            self.server_url, 'samlValidate',
        )
        request = Request(
            saml_validate_url + '?' + urllib_parse.urlencode(params),
            self.get_saml_assertion(ticket),
            headers,
        )
        return urllib_request.urlopen(request) 

def upload_disk(self, fileItem, lease, host):
        """
        Upload an individual disk. Passes the file handle of the
        disk directly to the urlopen request.
        """
        ovffile = self.get_disk(fileItem, lease)
        if ovffile is None:
            return
        deviceUrl = self.get_device_url(fileItem, lease)
        url = deviceUrl.url.replace('*', host)
        headers = {'Content-length': get_tarfile_size(ovffile)}
        if hasattr(ssl, '_create_unverified_context'):
            sslContext = ssl._create_unverified_context()
        else:
            sslContext = None
        req = Request(url, ovffile, headers)
        urlopen(req, context=sslContext) 

def DetectGce():
    """Determine whether or not we're running on GCE.

    This is based on:
      https://cloud.google.com/compute/docs/metadata#runninggce

    Returns:
      True iff we're running on a GCE instance.
    """
    metadata_url = 'http://{}'.format(
        os.environ.get('GCE_METADATA_ROOT', 'metadata.google.internal'))
    try:
        o = urllib_request.build_opener(urllib_request.ProxyHandler({})).open(
            urllib_request.Request(
                metadata_url, headers={'Metadata-Flavor': 'Google'}))
    except urllib_error.URLError:
        return False
    return (o.getcode() == http_client.OK and
            o.headers.get('metadata-flavor') == 'Google') 

def grab_artifact_from_jenkins(**context):
        """
        Grab an artifact from the previous job
        The python-jenkins library doesn't expose a method for that
        But it's totally possible to build manually the request for that
        """
        hook = JenkinsHook("your_jenkins_connection")
        jenkins_server = hook.get_jenkins_server()
        url = context['task_instance'].xcom_pull(task_ids='trigger_job')
        # The JenkinsJobTriggerOperator store the job url in the xcom variable corresponding to the task
        # You can then use it to access things or to get the job number
        # This url looks like : http://jenkins_url/job/job_name/job_number/
        url = url + "artifact/myartifact.xml"  # Or any other artifact name
        request = Request(url)
        response = jenkins_server.jenkins_open(request)
        return response  # We store the artifact content in a xcom variable for later use 

def _download_all_metadata(url):
    """
    Downloads the json file that contains all of the metadata for a specific file type (read:
    audio files, benchmark files, or trained models) that is on the EFZ server. This is retrieved
    from one of following three URLs (which are stored in nussl.constants):
    NUSSL_EFZ_AUDIO_METADATA_URL, NUSSL_EFZ_BENCHMARK_METADATA_URL, or NUSSL_EFZ_MODEL_METADATA_URL.

    Args:
        url (str):  URL for the EFZ server that has metadata. One of these three:
            NUSSL_EFZ_AUDIO_METADATA_URL, NUSSL_EFZ_BENCHMARK_METADATA_URL, or
            NUSSL_EFZ_MODEL_METADATA_URL.

    Returns:
        (list): List of dicts with metadata for the desired file type.

    """
    request = Request(url)

    # Make sure to get the newest data
    request.add_header('Pragma', 'no-cache')
    request.add_header('Cache-Control', 'max-age=0')
    try:
        return json.loads(urlopen(request).read())
    except:
        raise NoConnectivityError("Can't connect to internet") 

def read(self, amount):
        start = self.offset
        end = self.offset + amount - 1
        req = Request(self.url,
                      headers={'Range': 'bytes=%d-%d' % (start, end)})
        r = urlopen(req)
        self.offset += amount
        result = r.read(amount)
        r.close()
        return result

    # A slightly more accurate percentage 

def request_gce_metadata(path):
  req = Request('http://metadata/computeMetadata/v1/%s' % path,
                headers={'Metadata-Flavor': 'Google'})
  resp = urlopen(req, timeout=2)
  return tf.compat.as_str(resp.read()) 

def __init__(self, url, data=None, headers={}, method=None):
            if isinstance(Request, type):
                super(Request, self).__init__(url, data, headers)
            else:
                self.superclass.__init__(self, url, data, headers)
            if method is not None:
                self.method = method 

def test_copy_dest(self):
        # No COPY in swiftclient. Using urllib instead...
        url = os.path.join(self.url, self.container, self.storlet_file)
        objname = self.storlet_file + '-copy-ex'
        headers = {'X-Auth-Token': self.token,
                   'X-Run-Storlet': self.storlet_name,
                   'Destination': '%s/%s' % (self.container, objname)}
        headers.update(self.additional_headers)
        req = Request(url, headers=headers)
        req.get_method = lambda: 'COPY'
        conn = urlopen(req, timeout=10)

        self.assertEqual(201, conn.getcode())
        self.assertEqual('%s/%s' % (self.container, self.storlet_file),
                         conn.info()['x-storlet-generated-from'])
        self.assertEqual(self.acct,
                         conn.info()['x-storlet-generated-from-account'])
        self.assertIn('x-storlet-generated-from-last-modified', conn.info())

        headers = client.head_object(self.url, self.token,
                                     self.container, objname)
        self.assertEqual(str(len(self.content)), headers['content-length'])

        resp = dict()
        client.delete_object(
            self.url, self.token, self.container, objname,
            response_dict=resp)
        self.assertEqual(204, resp['status']) 

def _get_content(url):
  """Opens the url and loads the response into json."""
  logging.info('opening url %s', url)
  req = request.Request(url)
  resp = request.urlopen(req)
  resp_text = _as_text(resp.read())
  logging.info('response text = %s', resp_text)
  return json.loads(resp_text) 

def fetch_tile(x, y, z, tileserver):
    """Fetch tile (x, y) at zoom level z from OpenStreetMap's servers.

    Return a PIL image.

    """

    url = get_url(x, y, z, tileserver)
    req = Request(url, data=None, headers={'User-Agent': 'smopy'})
    png = BytesIO(urlopen(req).read())
    img = Image.open(png)
    img.load()
    return img 

def authenticate(self, url, params):
        """
        returns a request object ready to be issued to the Mixpanel API
        """
        request_url = '{base_url}?{encoded_params}'.format(
            base_url=url,
            encoded_params=_unicode_urlencode(params)
        )
        request_headers = {
            'Authorization': 'Basic ' + _totext(base64.standard_b64encode(_tobytes("{}:".format(self.client.api_secret))))
        }
        return url_request.Request(request_url, headers=request_headers) 

def get(url, data, callback, dataType):
    if dataType == 'jsonp':
        data = copy.deepcopy(data)
        data['callback'] = 'callback'
        _data = parse.urlencode(data)
        req = request.Request(url + '?' + _data if _data else url)
        res = request.urlopen(req, timeout=5)  # TODO: remove hardcoded timeout
        assert 200 == res.getcode()
        page = res.read().decode('utf-8').strip()
        assert page.startswith(data['callback'] + '({') and page.endswith('})')
        page = page[len(data['callback']) + 1:-1]
        page = json.loads(page)
        if callback:
            page = callback(page)
        return page
    elif dataType == 'raw':
        data = parse.urlencode(data)
        req = request.Request(url + '?' + data if data else url)
        res = request.urlopen(req, timeout=5)
        assert 200 == res.getcode()
        page = res.read().decode('utf-8')
        if callback:
            page = callback(page)
        return page
    else:
        raise NotImplementedError 

def _auth_checklogin(ipv):
    req = request.Request(
            'https://auth{:d}.tsinghua.edu.cn/ac_detect.php?ac_id=1'
            .format(ipv)
    )
    res = request.urlopen(req, timeout=5)
    assert 200 == res.getcode()
    url = res.geturl()
    username = parse.parse_qs(parse.urlparse(url).query).get('username')
    if not username:
        return {}
    else:
        return {
            'username': username[0],
        } 

def fetch_queries(
    host,
    prefix,
    auth_key,
    queries,
    from_param,
    until_param,
    timeout_s,
    threshold,
    progress_cb,
):
    """Return a list of HostResult."""
    host_result = HostResult(host)
    for n, query in enumerate(queries):
        url = _get_url_from_query(host, prefix, query, from_param, until_param)
        request = Request(url, auth_key, timeout_s)
        try:
            diffable_targets, time_s = request.execute()
            host_result.add_time_s(query, time_s)
            host_result.add_diffable_query(
                DiffableQuery(query, diffable_targets, threshold)
            )
        except RequestError as e:
            host_result.add_error(query, e)
            host_result.add_diffable_query(DiffableQuery(query, [], threshold))

        progress_cb(n)
    return host_result 

def _prepare(self, url, auth_key, data=None):
        """Create an http request."""
        headers = {}
        if auth_key is not None:
            headers["Authorization"] = "Basic %s" % auth_key
        request = urllib.Request(url, data, headers)
        return request 

def __init__(self, url, auth_key, timeout_s, data=None):
        """Create a Request."""
        self._request = self._prepare(url, auth_key, data=data)
        self._timeout_s = timeout_s 

def _query_iedb(request_values, url):
    """
    Call into IEDB's web API for MHC binding prediction using request dictionary
    with fields:
        - "method"
        - "length"
        - "sequence_text"
        - "allele"

    Parse the response into a DataFrame.
    """
    data = urlencode(request_values)
    req = Request(url, data.encode("ascii"))
    response = urlopen(req).read()
    return _parse_iedb_response(response) 

def ExpandRelativePath(method_config, params, relative_path=None):
    """Determine the relative path for request."""
    path = relative_path or method_config.relative_path or ''

    for param in method_config.path_params:
        param_template = '{%s}' % param
        # For more details about "reserved word expansion", see:
        #   http://tools.ietf.org/html/rfc6570#section-3.2.2
        reserved_chars = ''
        reserved_template = '{+%s}' % param
        if reserved_template in path:
            reserved_chars = _RESERVED_URI_CHARS
            path = path.replace(reserved_template, param_template)
        if param_template not in path:
            raise exceptions.InvalidUserInputError(
                'Missing path parameter %s' % param)
        try:
            # TODO(craigcitro): Do we want to support some sophisticated
            # mapping here?
            value = params[param]
        except KeyError:
            raise exceptions.InvalidUserInputError(
                'Request missing required parameter %s' % param)
        if value is None:
            raise exceptions.InvalidUserInputError(
                'Request missing required parameter %s' % param)
        try:
            if not isinstance(value, six.string_types):
                value = str(value)
            path = path.replace(param_template,
                                urllib_parse.quote(value.encode('utf_8'),
                                                   reserved_chars))
        except TypeError as e:
            raise exceptions.InvalidUserInputError(
                'Error setting required parameter %s to value %s: %s' % (
                    param, value, e))
    return path 

def send(self, payload):
        path, content_type = self._get_path_content_type(payload)
        url = "http://{}:{}{}".format(self.address, self.port, path)

        req = Request(url, payload, {"Content-Type": content_type})
        response = urlopen(req)

        assert response.getcode() == 202 

def assertUrl(self, url):

            prefix, path = url.split(':', 1)
            if prefix == 'file':
                self.assertTrue(os.path.exists(path))
            else:
                try:
                    urlopen(Request(url))
                except:
                    self.fail() 

def _request(self, req_body=None, method=None,
                 header=constants.CONTENT_TYPE_URLENCODE):
        req = url_request.Request(self._url, req_body.encode(), header)
        if method not in (None, 'GET', 'POST'):
            req.get_method = lambda: method
        self._http_log_req(req)
        try:
            resp = self.url_opener.open(req)
            resp_body = resp.read()
            self._http_log_resp(resp, resp_body)
        except url_error.HTTPError as http_err:
            if '403' == six.text_type(http_err.code):
                raise exception.NotAuthorized()
            else:
                err = {'errorCode': -1,
                       'httpStatusCode': http_err.code,
                       'messages': six.text_type(http_err),
                       'request': req_body}
                msg = (_("The request is invalid. Reason: %(reason)s") %
                       {'reason': err})
                raise exception.ManilaException(message=msg)

        return resp_body 

def _do_setup(self):
        credential = ('user=' + self.username
                      + '&password=' + self.password
                      + '&Login=Login')
        req = url_request.Request(self.auth_url, credential.encode(),
                                  constants.CONTENT_TYPE_URLENCODE)
        resp = self.url_opener.open(req)
        resp_body = resp.read()
        self._http_log_resp(resp, resp_body) 

def download(baseurl, parameters={}, headers={}):
    '''Download Data from an url and returns it as a String
    @param baseurl Url to download from (e.g. http://www.google.com)
    @param parameters Parameter dict to be encoded with url
    @param headers Headers dict to pass with Request
    @returns String of data from URL
    '''
    url = '?'.join([baseurl, urlencode(parameters)])
    log.debug('Downloading: ' + url)
    data = ""
    for _ in range(MAX_RETRIES):
        try:
            req = Request(url, headers=headers)
            req.add_header(USER_AGENT, USER_AGENT_STRING)
            response = urlopen(req)
            if six.PY2:
                data = response.read()
            else:
                data = response.read().decode('utf-8')
            response.close()
            break
        except Exception as err:
            if not isinstance(err, URLError):
                log.debug("Error %s during HTTP Request, abort", repr(err))
                raise  # propagate non-URLError
            log.debug("Error %s during HTTP Request, retrying", repr(err))
    else:
        raise
    return data 

def get_json(baseurl, parameters={}, headers={}):
    '''Download Data from an URL and returns it as JSON
    @param url Url to download from
    @param parameters Parameter dict to be encoded with url
    @param headers Headers dict to pass with Request
    @returns JSON Object with data from URL
    '''
    jsonString = download(baseurl, parameters, headers)
    jsonDict = json.loads(jsonString)
    log.debug(json.dumps(jsonDict, indent=4, sort_keys=True))
    return jsonDict 

def request_gce_metadata(path):
  req = Request('http://metadata/computeMetadata/v1/%s' % path,
                headers={'Metadata-Flavor': 'Google'})
  resp = urlopen(req, timeout=2)
  return tf.compat.as_str(resp.read()) 

def request_gce_metadata(path):
  req = Request('http://metadata/computeMetadata/v1/%s' % path,
                headers={'Metadata-Flavor': 'Google'})
  resp = urlopen(req, timeout=2)
  return tf.compat.as_str(resp.read())