Python six.moves.urllib.parse.urlunsplit() Examples

def _add_query_parameters(base_url, name_value_pairs):
    """Add one query parameter to a base URL.

    :type base_url: string
    :param base_url: Base URL (may already contain query parameters)

    :type name_value_pairs: list of (string, string) tuples.
    :param name_value_pairs: Names and values of the query parameters to add

    :rtype: string
    :returns: URL with additional query strings appended.
    """
    if len(name_value_pairs) == 0:
        return base_url

    scheme, netloc, path, query, frag = urlsplit(base_url)
    query = parse_qsl(query)
    query.extend(name_value_pairs)
    return urlunsplit((scheme, netloc, path, urlencode(query), frag)) 

def remove_trailing_version_from_href(href):
    """Removes the api version from the href.

    Given: 'http://www.masakari.com/ha/v1.1'
    Returns: 'http://www.masakari.com/ha'

    Given: 'http://www.masakari.com/v1.1'
    Returns: 'http://www.masakari.com'

    """
    parsed_url = urlparse.urlsplit(href)
    url_parts = parsed_url.path.rsplit('/', 1)

    # NOTE: this should match vX.X or vX
    expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)')
    if not expression.match(url_parts.pop()):
        LOG.debug('href %s does not contain version', href)
        raise ValueError(_('href %s does not contain version') % href)

    new_path = url_join(*url_parts)
    parsed_url = list(parsed_url)
    parsed_url[2] = new_path
    return urlparse.urlunsplit(parsed_url) 

def _build_url(self, url):
        """Build a complete URL from a path by substituting in session parameters."""
        components = urlsplit(url)

        domain = components.netloc or self._settings['domain']

        # Add port if a non-standard port was specified
        if self._settings['port'] is not None and ':' not in domain:
            domain = '{}:{}'.format(domain, self._settings['port'])

        return urlunsplit([
            components.scheme or self._settings['protocol'],
            domain,
            components.path,
            components.query,
            components.fragment
        ]) 

def _get_subjurisdictions_url(self):
        """
        Returns a URL for the county detail page, which lists URLs for
        each of the counties in a state. If original jurisdiction is
        not a state, returns None.
        """
        if self.level != 'state':
            return None
        elif 'Web01/' in self.url:
            return None
        else:
            newpath = '/'.join(self.parsed_url.path.split('/')[:-1]) + '/select-county.html'
            parts = (
                self.parsed_url.scheme,
                self.parsed_url.netloc,
                newpath,
                self.parsed_url.query,
                self.parsed_url.fragment,
            )
            return parse.urlunsplit(parts) 

def update_query_parameters(url, query_parameters):
    """
    Return url with updated query parameters.

    Arguments:
        url (str): Original url whose query parameters need to be updated.
        query_parameters (dict): A dictionary containing query parameters to be added to course selection url.

    Returns:
        (slug): slug identifier for the identity provider that can be used for identity verification of
            users associated the enterprise customer of the given user.

    """
    scheme, netloc, path, query_string, fragment = urlsplit(url)
    url_params = parse_qs(query_string)

    # Update url query parameters
    url_params.update(query_parameters)

    return urlunsplit(
        (scheme, netloc, path, urlencode(sorted(url_params.items()), doseq=True), fragment),
    ) 

def get_current_ver(cls, election_url):
        election_url_parts = parse.urlsplit(cls._url_ensure_trailing_slash(election_url))
        if 'Web02' in election_url:
            cls.parsed_url = election_url_parts._replace(path="/".join(election_url_parts.path.split('/')[:3]) + "/current_ver.txt", fragment='')
            election_url_parts =  cls.parsed_url
        else:
            election_url_parts = election_url_parts._replace(path=election_url_parts.path + "current_ver.txt")

        current_ver_url = parse.urlunsplit(election_url_parts)

        current_ver_response = requests.get(current_ver_url)

        try:
            current_ver_response.raise_for_status()
        except requests.exceptions.HTTPError:
            return None

        return current_ver_response.text 

def __attrs_post_init__(self):
        # type: () -> None
        if not self.uri:
            if self.path:
                self.uri = pip_shims.shims.path_to_url(self.path)
        if self.uri is not None:
            split = urllib_parse.urlsplit(self.uri)
            scheme, rest = split[0], split[1:]
            vcs_type = ""
            if "+" in scheme:
                vcs_type, scheme = scheme.split("+", 1)
                vcs_type = "{0}+".format(vcs_type)
            new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",))
            new_uri = "{0}{1}".format(vcs_type, new_uri)
            self.uri = new_uri 

def _update_link_prefix(self, orig_url, prefix):
        if not prefix:
            return orig_url
        url_parts = list(urlparse.urlsplit(orig_url))
        prefix_parts = list(urlparse.urlsplit(prefix))
        url_parts[0:2] = prefix_parts[0:2]
        url_parts[2] = prefix_parts[2] + url_parts[2]
        return urlparse.urlunsplit(url_parts).rstrip('/') 

def proxy_request(self, req):
        headers = self.get_headers(req)
        url = urlparse.urlunsplit((
            self.get_scheme(req),
            self.get_host(req),
            self.get_path_info(req),
            self.get_query_string(req),
            ''))
        h = self.create_http_client(req)
        resp, content = h.request(
            url,
            method=req.method,
            headers=headers,
            body=req.body
        )
        if resp.status == 200:
            LOG.debug(str(resp))
            return self.create_response(req, resp, content)
        elif resp.status == 403:
            LOG.warning(
                'The remote metadata server responded with Forbidden. This '
                'response usually occurs when shared secrets do not match.')
            return webob.exc.HTTPForbidden()
        elif resp.status == 400:
            return webob.exc.HTTPBadRequest()
        elif resp.status == 404:
            return webob.exc.HTTPNotFound()
        elif resp.status == 409:
            return webob.exc.HTTPConflict()
        elif resp.status == 500:
            msg = (
                'Remote metadata server experienced an internal server error.'
            )
            LOG.warning(msg)
            explanation = six.text_type(msg)
            return webob.exc.HTTPInternalServerError(explanation=explanation)
        else:
            raise Exception(_('Unexpected response code: %s') % resp.status) 

def _update_link_prefix(self, orig_url, prefix):
        if not prefix:
            return orig_url
        url_parts = list(parse.urlsplit(orig_url))
        prefix_parts = list(parse.urlsplit(prefix))
        url_parts[0:2] = prefix_parts[0:2]
        return parse.urlunsplit(url_parts) 

def get_url_with_credentials(url, username, password):
    parts = parse.urlsplit(url)
    # Remove previous credentials if set
    netloc = parts.netloc[parts.netloc.find('@') + 1:]
    netloc = "%s:%s@%s" % (
        quote_url(username), quote_url(password or ''), netloc)
    parts = parts._replace(netloc=netloc)
    return parse.urlunsplit(parts) 

def get_authorization_url(self, redirect_url):
        """
        Get the authorization url based on the client id and the redirect url passed in

        :param redirect_url:
            An HTTPS URI or custom URL scheme where the response will be redirected. Optional if the redirect URI is
            registered with Box already.
        :type redirect_url:
            `unicode` or None
        :return:
            A tuple of the URL of Box's authorization page and the CSRF token.
            This is the URL that your application should forward the user to in first leg of OAuth 2.
        :rtype:
            (`unicode`, `unicode`)
        """
        csrf_token = self._get_state_csrf_token()
        # For the query string parameters, use a sequence of two-element
        # tuples, rather than a dictionary, in order to get a consistent and
        # predictable order of parameters in the output of `urlencode()`.
        params = [
            ('state', csrf_token),
            ('response_type', 'code'),
            ('client_id', self._client_id),
        ]
        if redirect_url:
            params.append(('redirect_uri', redirect_url))
        # `urlencode()` doesn't work with non-ASCII unicode characters, so
        # encode the parameters as ASCII bytes.
        params = [(key.encode('utf-8'), value.encode('utf-8')) for (key, value) in params]
        query_string = urlencode(params)
        return urlunsplit(('', '', self._api_config.OAUTH2_AUTHORIZE_URL, query_string, '')), csrf_token 

def long_poll_url(test_url, expected_stream_type_params):
    return urlunsplit(('', '', test_url, urlencode(expected_stream_type_params), '')) 

def relative_url(url):
    parts = urlsplit(url)
    return urlunsplit(('', '') + parts[2:]) 

def get_url(path, host, port, method="http"):
    """
    make url from path, host and port

    :param method: str
    :param path: str, path within the request, e.g. "/api/version"
    :param host: str
    :param port: str or int
    :return: str
    """
    return urlunsplit(
        (method, "%s:%s" % (host, port), path, "", "")
    ) 

def _get_url_base(cls, url):
        """
        Return the base of a URL
        """
        s = urlsplit(url)
        return urlunsplit([s.scheme, s.netloc, '', '', '']) 

def _get_url_base(self, url):
        """
        Return the base of a URL
        """
        s = urlsplit(url)
        return urlunsplit([s.scheme, s.netloc, '', '', '']) 

def remove_version_from_href(href):
    """Removes the first api version from the href.

    Given: 'http://manila.example.com/v1.1/123'
    Returns: 'http://manila.example.com/123'

    Given: 'http://www.manila.com/v1.1'
    Returns: 'http://www.manila.com'

    Given: 'http://manila.example.com/share/v1.1/123'
    Returns: 'http://manila.example.com/share/123'

    """
    parsed_url = parse.urlsplit(href)
    url_parts = parsed_url.path.split('/')

    # NOTE: this should match vX.X or vX
    expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)')
    for x in range(len(url_parts)):
        if expression.match(url_parts[x]):
            del url_parts[x]
            break

    new_path = '/'.join(url_parts)

    if new_path == parsed_url.path:
        msg = 'href %s does not contain version' % href
        LOG.debug(msg)
        raise ValueError(msg)

    parsed_url = list(parsed_url)
    parsed_url[2] = new_path
    return parse.urlunsplit(parsed_url) 

def get_latest_summary_url(cls, election_url):
        election_url = cls._url_ensure_trailing_slash(election_url)
        current_ver = cls.get_current_ver(election_url)

        # If we don't have current_ver, we can't determine a summary URL.
        if current_ver is None:
            return None

        if 'Web02' in election_url:
            election_url_parts = parse.urlsplit(election_url)
            election_url_parts = election_url_parts._replace(path="/".join(election_url_parts.path.split('/')[:3]), fragment='')
        else:
            election_url_parts = parse.urlsplit(election_url)

        new_paths = [
            election_url_parts.path + '/' + current_ver + "/json/en/summary.json",
            election_url_parts.path + current_ver + "/Web01/en/summary.html",
            election_url_parts.path + current_ver + "/en/summary.html",
        ]

        for new_path in new_paths:
            latest_summary_url_parts = election_url_parts._replace(path=new_path)

            latest_summary_url = parse.urlunsplit(latest_summary_url_parts)

            latest_summary_url_response = requests.get(latest_summary_url)

            try:
                latest_summary_url_response.raise_for_status()
            except requests.exceptions.HTTPError:
                continue

            return latest_summary_url

        # If none of the expected paths succeed, return None.
        return None 

def _url_ensure_trailing_slash(cls, url):
        url_parts = parse.urlsplit(url)

        # Ensure the incoming URL ends in a slash.
        if not url_parts.path.endswith("/"):
            url_parts = url_parts._replace(path=url_parts.path + "/")

        return parse.urlunsplit(url_parts) 

def __attrs_post_init__(self):
        # type: () -> None
        if not self.uri:
            if self.path:
                self.uri = pip_shims.shims.path_to_url(self.path)
        if self.uri is not None:
            split = urllib_parse.urlsplit(self.uri)
            scheme, rest = split[0], split[1:]
            vcs_type = ""
            if "+" in scheme:
                vcs_type, scheme = scheme.split("+", 1)
                vcs_type = "{0}+".format(vcs_type)
            new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",))
            new_uri = "{0}{1}".format(vcs_type, new_uri)
            self.uri = new_uri 

def remove_query_param(url, key):
    (scheme, netloc, path, query, fragment) = parse.urlsplit(six.text_type(url))
    query_dict = parse.parse_qs(query, keep_blank_values=True)
    query_dict.pop(key, None)
    query = parse.urlencode(sorted(list(query_dict.items())), doseq=True)
    return parse.urlunsplit((scheme, netloc, path, query, fragment)) 

def replace_query_param(url, key, val):
    (scheme, netloc, path, query, fragment) = parse.urlsplit(six.text_type(url))
    query_dict = parse.parse_qs(query, keep_blank_values=True)
    query_dict[six.text_type(key)] = [six.text_type(val)]
    query = parse.urlencode(sorted(list(query_dict.items())), doseq=True)
    return parse.urlunsplit((scheme, netloc, path, query, fragment)) 

def get_root_uri(uri):
    """Return root URI - strip query and fragment."""
    chunks = urlsplit(uri)
    return urlunsplit((chunks.scheme, chunks.netloc, chunks.path, '', '')) 

def _proxy_request(self, req, requester):
        headers = self._build_proxy_request_headers(requester)
        nova_ip_port = '%s:%s' % (CONF.metadata.nova_metadata_ip,
                                  CONF.metadata.nova_metadata_port)
        url = urlparse.urlunsplit((
            CONF.metadata.nova_metadata_protocol,
            nova_ip_port,
            req.path_info,
            req.query_string,
            ''))

        h = httplib2.Http(
            ca_certs=CONF.metadata.auth_ca_cert,
            disable_ssl_certificate_validation=(
                    CONF.metadata.nova_metadata_insecure)
        )
        if (CONF.metadata.nova_client_cert and
                CONF.metadata.nova_client_priv_key):
            h.add_certificate(CONF.metadata.nova_client_priv_key,
                              CONF.metadata.nova_client_cert,
                              nova_ip_port)
        resp, content = h.request(url, method=req.method, headers=headers,
                                  body=req.body)

        if resp.status == 200:
            LOG.debug(str(resp))
            req.response.content_type = resp['content-type']
            req.response.body = content
            return req.response
        elif resp.status == 403:
            LOG.warning(
                'The remote metadata server responded with Forbidden. This '
                'response usually occurs when shared secrets do not match.'
            )
            return webob.exc.HTTPForbidden()
        elif resp.status == 400:
            return webob.exc.HTTPBadRequest()
        elif resp.status == 404:
            return webob.exc.HTTPNotFound()
        elif resp.status == 409:
            return webob.exc.HTTPConflict()
        elif resp.status == 500:
            msg = _(
                'Remote metadata server experienced an internal server error.'
            )
            LOG.warning(msg)
            return webob.exc.HTTPInternalServerError(
                explanation=six.text_type(msg))
        else:
            raise Exception(_('Unexpected response code: %s') % resp.status) 

def _fuzzy_query_call(self, query):
    # imports added here for brozzler
    from pywb.utils.loaders import to_native_str
    from six.moves.urllib.parse import urlsplit, urlunsplit

    matched_rule = None

    urlkey = to_native_str(query.key, 'utf-8')
    url = query.url
    filter_ = query.filters
    output = query.output

    for rule in self.rules.iter_matching(urlkey):
        m = rule.regex.search(urlkey)
        if not m:
            continue

        matched_rule = rule

        groups = m.groups()
        for g in groups:
            for f in matched_rule.filter:
                filter_.append(f.format(g))

        break

    if not matched_rule:
        return None

    repl = '?'
    if matched_rule.replace:
        repl = matched_rule.replace

    inx = url.find(repl)
    if inx > 0:
        url = url[:inx + len(repl)]

    # begin brozzler changes
    if matched_rule.match_type == 'domain':
        orig_split_url = urlsplit(url)
        # remove the subdomain, path, query and fragment
        host = orig_split_url.netloc.split('.', 1)[1]
        new_split_url = (orig_split_url.scheme, host, '', '', '')
        url = urlunsplit(new_split_url)
    # end brozzler changes

    params = query.params
    params.update({'url': url,
                   'matchType': matched_rule.match_type,
                   'filter': filter_})

    if 'reverse' in params:
        del params['reverse']

    if 'closest' in params:
        del params['closest']

    if 'end_key' in params:
        del params['end_key']

    return params 

def __init__(self, orig_url):
        import re
        import six

        from six.moves.urllib.parse import urlsplit, urlunsplit
        from six.moves.urllib.parse import quote_plus, quote, unquote_plus

        from pywb.utils.loaders import to_native_str
        from pywb.rewrite.wburl import WbUrl

        pywb.rewrite.wburl.BaseWbUrl.__init__(self)

        if six.PY2 and isinstance(orig_url, six.text_type):
            orig_url = orig_url.encode('utf-8')
            orig_url = quote(orig_url)

        self._original_url = orig_url

        if not self._init_query(orig_url):
            if not self._init_replay(orig_url):
                raise Exception('Invalid WbUrl: ', orig_url)

        new_uri = WbUrl.to_uri(self.url)

        self._do_percent_encode = True

        self.url = new_uri

        # begin brozzler changes
        if (self.url.startswith('urn:') or self.url.startswith('screenshot:')
                or self.url.startswith('thumbnail:')):
            return
        # end brozzler changes

        # protocol agnostic url -> http://
        # no protocol -> http://
        #inx = self.url.find('://')
        inx = -1
        m = self.SCHEME_RX.match(self.url)
        if m:
            inx = m.span(1)[0]

        #if inx < 0:
            # check for other partially encoded variants
        #    m = self.PARTIAL_ENC_RX.match(self.url)
        #    if m:
        #        len_ = len(m.group(0))
        #        self.url = (urllib.unquote_plus(self.url[:len_]) +
        #                    self.url[len_:])
        #        inx = self.url.find(':/')

        if inx < 0:
            self.url = self.DEFAULT_SCHEME + self.url
        else:
            inx += 2
            if inx < len(self.url) and self.url[inx] != '/':
                self.url = self.url[:inx] + '/' + self.url[inx:] 

def _cs_request(self, url, method, **kwargs):
        if not self.management_url:
            self.authenticate()
        if url is None:
            # To get API version information, it is necessary to GET
            # a nova endpoint directly without "v2/<tenant-id>".
            magic_tuple = parse.urlsplit(self.management_url)
            scheme, netloc, path, query, frag = magic_tuple
            path = re.sub(r'v[1-9]/[a-z0-9]+$', '', path)
            url = parse.urlunsplit((scheme, netloc, path, None, None))
        else:
            if self.service_catalog:
                url = self.get_service_url(self.service_type) + url
            else:
                # NOTE(melwitt): The service catalog is not available
                #                when bypass_url is used.
                url = self.management_url + url

        # Perform the request once. If we get a 401 back then it
        # might be because the auth token expired, so try to
        # re-authenticate and try again. If it still fails, bail.
        try:
            kwargs.setdefault('headers', {})['X-Auth-Token'] = self.auth_token
            kwargs['headers']['Content-Type'] = 'application/json'
            if self.projectid:
                kwargs['headers']['X-Auth-Project-Id'] = self.projectid

            resp, body = self._time_request(url, method, **kwargs)
            return resp, body
        except exceptions.Unauthorized as e:
            try:
                # first discard auth token, to avoid the possibly expired
                # token being re-used in the re-authentication attempt
                self.unauthenticate()
                # overwrite bad token
                self.keyring_saved = False
                self.authenticate()
                kwargs['headers']['X-Auth-Token'] = self.auth_token
                resp, body = self._time_request(url, method, **kwargs)
                return resp, body
            except exceptions.Unauthorized:
                raise e 

def _cs_request(self, url, method, **kwargs):
        if not self.management_url:
            self.authenticate()
        if url is None:
            # To get API version information, it is necessary to GET
            # a dh endpoint directly without "v2/<tenant-id>".
            magic_tuple = parse.urlsplit(self.management_url)
            scheme, netloc, path, query, frag = magic_tuple
            path = re.sub(r'v[1-9]/[a-z0-9]+$', '', path)
            url = parse.urlunsplit((scheme, netloc, path, None, None))
        else:
            if self.service_catalog:
                url = self.get_service_url(self.service_type) + url
            else:
                # NOTE(melwitt): The service catalog is not available
                #                when bypass_url is used.
                url = self.management_url + url

        # Perform the request once. If we get a 401 back then it
        # might be because the auth token expired, so try to
        # re-authenticate and try again. If it still fails, bail.
        try:
            kwargs.setdefault('headers', {})['X-Auth-Token'] = self.auth_token
            kwargs['headers']['Content-Type'] = 'application/json'
            if self.projectid:
                kwargs['headers']['X-Auth-Project-Id'] = self.projectid

            resp, body = self._time_request(url, method, **kwargs)
            return resp, body
        except exceptions.Unauthorized as e:
            try:
                # first discard auth token, to avoid the possibly expired
                # token being re-used in the re-authentication attempt
                self.unauthenticate()
                # overwrite bad token
                self.keyring_saved = False
                self.authenticate()
                kwargs['headers']['X-Auth-Token'] = self.auth_token
                resp, body = self._time_request(url, method, **kwargs)
                return resp, body
            except exceptions.Unauthorized:
                raise e 

def _cs_request(self, url, method, **kwargs):
        if not self.management_url:
            self.authenticate()
        if url is None:
            # To get API version information, it is necessary to GET
            # a icgw endpoint directly without "v2/<tenant-id>".
            magic_tuple = parse.urlsplit(self.management_url)
            scheme, netloc, path, query, frag = magic_tuple
            path = re.sub(r'v[1-9]/[a-z0-9]+$', '', path)
            url = parse.urlunsplit((scheme, netloc, path, None, None))
        else:
            if self.service_catalog:
                url = self.get_service_url(self.service_type) + url
            else:
                # NOTE(melwitt): The service catalog is not available
                #                when bypass_url is used.
                url = self.management_url + url

        # Perform the request once. If we get a 401 back then it
        # might be because the auth token expired, so try to
        # re-authenticate and try again. If it still fails, bail.
        try:
            kwargs.setdefault('headers', {})['X-Auth-Token'] = self.auth_token
            kwargs['headers']['Content-Type'] = 'application/json'
            if self.projectid:
                kwargs['headers']['X-Auth-Project-Id'] = self.projectid

            resp, body = self._time_request(url, method, **kwargs)
            return resp, body
        except exceptions.Unauthorized as e:
            try:
                # first discard auth token, to avoid the possibly expired
                # token being re-used in the re-authentication attempt
                self.unauthenticate()
                # overwrite bad token
                self.keyring_saved = False
                self.authenticate()
                kwargs['headers']['X-Auth-Token'] = self.auth_token
                resp, body = self._time_request(url, method, **kwargs)
                return resp, body
            except exceptions.Unauthorized:
                raise e