Python six.moves.urllib.parse.urlunparse() Examples

def proxy_with_warning_page_mock(url, *args, **kwargs):
    cookies = kwargs.get('cookies') or {}
    proxy_cookie = cookies.get('proxy_cookie')
    url_parts = list(urlparse(url))
    query = dict(parse_qsl(url_parts[4]))
    if proxy_cookie and query.get('proxyapproved') == 'true':
        del query['proxyapproved']
        url_parts[4] = urlencode(query)
        return standalone_requests_get_mock(urlunparse(url_parts), *args[1:], **kwargs)
    else:
        # Display the html warning page with the redirect link
        query['proxyapproved'] = 'true'
        url_parts[4] = urlencode(query)
        with open(os.path.join(FIXTURE_DIR, 'html_warning_page'), 'r') as f:
            body = f.read().replace('$REDIRECT_URL$', urlunparse(url_parts))
            cookies['proxy_cookie'] = 'foo'
            return MockedResponse(body, 200, cookies) 

def validate_link(self, link, bookmark=False):
        """Checks if the given link can get correct data."""
        # removes the scheme and net location parts of the link
        url_parts = list(urlparse.urlparse(link))
        url_parts[0] = url_parts[1] = ''

        # bookmark link should not have the version in the URL
        if bookmark and url_parts[2].startswith(PATH_PREFIX):
            return False

        full_path = urlparse.urlunparse(url_parts)
        try:
            self.get_json(full_path, path_prefix='')
            return True
        except Exception:
            return False 

def __init__(self, connection_str, index_name="osprofiler-notifications",
                 project=None, service=None, host=None, conf=cfg.CONF,
                 **kwargs):
        """Elasticsearch driver for OSProfiler."""

        super(ElasticsearchDriver, self).__init__(connection_str,
                                                  project=project,
                                                  service=service,
                                                  host=host,
                                                  conf=conf,
                                                  **kwargs)
        try:
            from elasticsearch import Elasticsearch
        except ImportError:
            raise exc.CommandError(
                "To use OSProfiler with ElasticSearch driver, "
                "please install `elasticsearch` library. "
                "To install with pip:\n `pip install elasticsearch`.")

        client_url = parser.urlunparse(parser.urlparse(self.connection_str)
                                       ._replace(scheme="http"))
        self.conf = conf
        self.client = Elasticsearch(client_url)
        self.index_name = index_name
        self.index_name_error = "osprofiler-notifications-error" 

def _new_location(self, old_location, url):
        store_name = old_location.store_name
        store_class = old_location.store_location.__class__
        image_id = old_location.image_id
        store_specs = old_location.store_specs
        # Note(sabari): The redirect url will have a scheme 'http(s)', but the
        # store only accepts url with scheme 'vsphere'. Thus, replacing with
        # store's scheme.
        parsed_url = urlparse.urlparse(url)
        new_url = parsed_url._replace(scheme='vsphere')
        vsphere_url = urlparse.urlunparse(new_url)
        return glance_store.location.Location(store_name,
                                              store_class,
                                              self.conf,
                                              uri=vsphere_url,
                                              image_id=image_id,
                                              store_specs=store_specs,
                                              backend=self.backend_group) 

def requote_uri(uri):
    """Requote uri if it contains non-ascii chars, spaces etc.
    """
    # To reduce tabulator import time
    import requests.utils
    if six.PY2:
        def url_encode_non_ascii(bytes):
            pattern = '[\x80-\xFF]'
            replace = lambda c: ('%%%02x' % ord(c.group(0))).upper()
            return re.sub(pattern, replace, bytes)
        parts = urlparse(uri)
        uri = urlunparse(
            part.encode('idna') if index == 1
            else url_encode_non_ascii(part.encode('utf-8'))
            for index, part in enumerate(parts))
    return requests.utils.requote_uri(uri) 

def get_commit_url(commit, pkg):
    try:
        upstream_url = parse.urlsplit(pkg["upstream"])

        if upstream_url.netloc == "git.openstack.org":
            commit_url = ("http",
                          upstream_url.netloc,
                          "/cgit%s/commit/?id=" % upstream_url.path,
                          "", "", "")
            commit_url = parse.urlunparse(commit_url)
        elif upstream_url.netloc == "github.com":
            commit_url = ("https",
                          upstream_url.netloc,
                          "%s/commit/" % upstream_url.path,
                          "", "", "")
            commit_url = parse.urlunparse(commit_url)
        else:
            # Fallback when no cgit URL can be defined
            commit_url = pkg["upstream"]
    except KeyError:
        # This should not happen, but pkg['upstream'] may not be present
        # after some error in the gitrepo driver
        commit_url = ''

    return commit_url 

def __init__(self, conf):
        self.conf = conf
        db_name = 'aodh_%s' % uuidutils.generate_uuid(dashed=False)
        import sqlalchemy
        self._engine = sqlalchemy.create_engine(conf.database.connection)
        self._conn = self._engine.connect()
        self._create_db(self._conn, db_name)
        self._conn.close()
        self._engine.dispose()
        parsed = list(urlparse.urlparse(conf.database.connection))
        # NOTE(jd) We need to set an host otherwise urlunparse() will not
        # construct a proper URL
        if parsed[1] == '':
            parsed[1] = 'localhost'
        parsed[2] = '/' + db_name
        self.url = urlparse.urlunparse(parsed) 

def create_server(conn):
    token = id_client.get_scoped_token_v3("user")
    _url = urlparse.urlunparse(CONF.syntribos.endpoint)
    endpoint = urlparse.urlunparse(
        (_url.scheme,
         _url.hostname + ":9292",
         _url.path,
         _url.params,
         _url.query,
         _url.fragment))
    _gc = GC(endpoint=endpoint, token=token)
    image = _gc.images.get(get_image_id())
    flavor = conn.flavors.get(get_flavor_id())
    server = conn.servers.create(
        name="test", flavor=flavor, image=image)

    return server.id 

def get_image_id():
    token = id_client.get_scoped_token_v3("user")
    _url = urlparse.urlparse(CONF.syntribos.endpoint)
    endpoint = urlparse.urlunparse(
        (_url.scheme,
         _url.hostname + ":9292",
         _url.path,
         _url.params,
         _url.query,
         _url.fragment))
    _gc = GC(endpoint=endpoint, token=token)
    image_ids = [image.id for image in _gc.images.list()]
    if not image_ids:
        image_ids.append(_gc.images.create(name="test"))

    return image_ids[-1] 

def __init__(
        self,
        host="localhost",
        port=9200,
        path="",
        scheme="http",
        user=None,
        password=None,
        context=None,
        **kwargs,
    ):
        netloc = f"{host}:{port}"
        path = path or "/"
        self.url = parse.urlunparse((scheme, netloc, path, None, None, None))
        self.context = context or {}
        self.closed = False
        self.cursors = []
        self.kwargs = kwargs
        # Subclass needs to initialize Elasticsearch
        self.es = None 

def censor_connect_string(connect_string):
    """
    Take a SQLAlchemy connect string and return a sanitized version
    that can be written to the log without disclosing the password.
    The password is replaced with "xxxx".
    In case any error occurs, return "<error when censoring connect string>"
    """
    try:
        parsed = urlparse(connect_string)
        if parsed.password is not None:
            # We need to censor the ``netloc`` attribute: user:pass@host
            _, host = parsed.netloc.rsplit("@", 1)
            new_netloc = u'{}:{}@{}'.format(parsed.username, 'xxxx', host)
            # Convert the URL to six components. netloc is component #1.
            splitted = list(parsed)
            splitted[1] = new_netloc
            return urlunparse(splitted)
        return connect_string
    except Exception:
        return "<error when censoring connect string>" 

def _ping_indexer(self, search_engine):
        # App Engine makes HTTP requests to tasks, so building a URL based on
        # the request will use HTTP instead of HTTPS, but we want to send search
        # engines HTTPS URLs.
        sitemap_url = self.build_absolute_uri('/global/sitemap')
        sitemap_url_parts = list(urlparse.urlparse(sitemap_url))
        sitemap_url_parts[0] = 'https'  # 0 is the index of the scheme part.
        sitemap_url = urlparse.urlunparse(sitemap_url_parts)
        ping_url = _INDEXER_MAP[search_engine] % urllib.parse.quote(sitemap_url)
        response = requests.get(ping_url)
        if response.status_code == 200:
            return True
        else:
            # TODO(nworden): Retry or email konbit-personfinder on failure.
            logging.error('Received %d pinging %s',
                          response.status_code, ping_url)
            return False 

def _decorate_request(self, filters, method, url, headers=None, body=None,
                          auth_data=None):
        if auth_data is None:
            auth_data = self.auth_data
        token, _ = auth_data
        base_url = self.base_url(filters=filters, auth_data=auth_data)
        # build authenticated request
        # returns new request, it does not touch the original values
        _headers = copy.deepcopy(headers) if headers is not None else {}
        _headers['X-Auth-Token'] = str(token)
        if url is None or url == "":
            _url = base_url
        else:
            # Join base URL and url, and remove multiple contiguous slashes
            _url = "/".join([base_url, url])
            parts = [x for x in urlparse.urlparse(_url)]
            parts[2] = re.sub("/{2,}", "/", parts[2])
            _url = urlparse.urlunparse(parts)
        # no change to method or body
        return str(_url), _headers, body 

def _build_url(url, _params):
    """Build the actual URL to use."""

    # Support for unicode domain names and paths.
    scheme, netloc, path, params, query, fragment = urlparse(url)
    netloc = netloc.encode('idna').decode('utf-8')
    if not path:
        path = '/'

    if six.PY2:
        if isinstance(scheme, six.text_type):
            scheme = scheme.encode('utf-8')
        if isinstance(netloc, six.text_type):
            netloc = netloc.encode('utf-8')
        if isinstance(path, six.text_type):
            path = path.encode('utf-8')
        if isinstance(params, six.text_type):
            params = params.encode('utf-8')
        if isinstance(query, six.text_type):
            query = query.encode('utf-8')
        if isinstance(fragment, six.text_type):
            fragment = fragment.encode('utf-8')

    enc_params = _encode_params(_params)
    if enc_params:
        if query:
            query = '%s&%s' % (query, enc_params)
        else:
            query = enc_params
    url = (urlunparse([scheme, netloc, path, params, query, fragment]))
    return url 

def _canonicalize_uri(uri):
    scheme, netloc, path, params, query, fragment = parse.urlparse(uri)
    if query:
        query = parse.urlencode(sorted(parse.parse_qsl(query)))
    return parse.urlunparse((scheme, netloc, path, params, query, fragment)) 

def get_websocket_url(url):
    parsed_url = urlparse(url)
    parts = list(parsed_url)
    if parsed_url.scheme == 'http':
        parts[0] = 'ws'
    elif parsed_url.scheme == 'https':
        parts[0] = 'wss'
    return urlunparse(parts) 

def get_url(url, headers=0, gid=0, sheet=None):
    parts = parse.urlparse(url)
    if parts.path.endswith('/edit'):
        path = parts.path[:-len('/edit')]
    else:
        path = parts.path
    path = '/'.join((path.rstrip('/'), 'gviz/tq'))

    qs = parse.parse_qs(parts.query)
    if 'headers' in qs:
        headers = int(qs['headers'][-1])
    if 'gid' in qs:
        gid = qs['gid'][-1]
    if 'sheet' in qs:
        sheet = qs['sheet'][-1]

    if parts.fragment.startswith('gid='):
        gid = parts.fragment[len('gid='):]

    args = OrderedDict()
    if headers > 0:
        args['headers'] = headers
    if sheet is not None:
        args['sheet'] = sheet
    else:
        args['gid'] = gid
    params = parse.urlencode(args)

    return parse.urlunparse(
        (parts.scheme, parts.netloc, path, None, params, None)) 

def set_query_params(url, params):
    """Set params in given query."""
    url_parts = parse.urlparse(url)
    url = parse.urlunparse((
        url_parts.scheme,
        url_parts.netloc,
        url_parts.path,
        url_parts.params,
        parse.urlencode(params),
        url_parts.fragment))
    return url 

def get_websocket_url(url):
    parsed_url = urlparse(url)
    parts = list(parsed_url)
    if parsed_url.scheme == 'http':
        parts[0] = 'ws'
    elif parsed_url.scheme == 'https':
        parts[0] = 'wss'
    return urlunparse(parts) 

def get_thumbnail_url(local_photo_url):
        parsed_url = list(urlparse.urlparse(local_photo_url))
        params_dict = dict(urlparse.parse_qsl(
            parsed_url[Person.URL_PARSE_QUERY_INDEX]))
        params_dict['thumb'] = 'true'
        parsed_url[Person.URL_PARSE_QUERY_INDEX] = urllib.urlencode(
            params_dict)
        return urlparse.urlunparse(parsed_url) 

def build_url(scheme, host, path='/', username=None, password=None, query_params=None):
    # type: (str, str, str, str, str, dict) -> str
    """Build an URL from individual parts. Make sure that parts are properly URL-encoded."""
    if username and password:
        netloc = '{}:{}@{}'.format(quote_plus(username), quote_plus(password), host)
    else:
        netloc = host

    path_params = ""
    query = urlencode(query_params or {})
    fragment = ""

    return urlunparse([scheme, netloc, path, path_params, query, fragment]) 

def add_headers(url, headers):
    parts = parse.urlparse(url)
    if parts.fragment.startswith('gid='):
        gid = parts.fragment[len('gid='):]
    else:
        gid = 0
    params = parse.urlencode(OrderedDict([('headers', headers), ('gid', gid)]))
    return parse.urlunparse(
        (parts.scheme, parts.netloc, parts.path, None, params, None)) 

def unparsed_urls():
    return st.builds(urllib_parse.urlunparse, urls()) 

def __init__(self, metric, variant, src, iter=0, **kwargs):
        self._url = src
        parts = urlparse(src)
        self._key = urlunparse(('', '', parts.path, parts.params, parts.query, parts.fragment))
        super(ImageEventNoUpload, self).__init__(metric, variant, iter=iter, **kwargs) 

def unparsed_urls():
    return st.builds(urllib_parse.urlunparse, urls()) 

def add_ssh_scheme_to_git_uri(uri):
    # type: (S) -> S
    """Cleans VCS uris from pip format"""
    if isinstance(uri, six.string_types):
        # Add scheme for parsing purposes, this is also what pip does
        if uri.startswith("git+") and "://" not in uri:
            uri = uri.replace("git+", "git+ssh://", 1)
            parsed = urlparse(uri)
            if ":" in parsed.netloc:
                netloc, _, path_start = parsed.netloc.rpartition(":")
                path = "/{0}{1}".format(path_start, parsed.path)
                uri = urlunparse(parsed._replace(netloc=netloc, path=path))
    return uri 

def strip_ssh_from_git_uri(uri):
    # type: (S) -> S
    """Return git+ssh:// formatted URI to git+git@ format"""
    if isinstance(uri, six.string_types):
        if "git+ssh://" in uri:
            parsed = urlparse(uri)
            # split the path on the first separating / so we can put the first segment
            # into the 'netloc' section with a : separator
            path_part, _, path = parsed.path.lstrip("/").partition("/")
            path = "/{0}".format(path)
            parsed = parsed._replace(
                netloc="{0}:{1}".format(parsed.netloc, path_part), path=path
            )
            uri = urlunparse(parsed).replace("git+ssh://", "git+", 1)
    return uri 

def get_files_server_host(cls, config=None):
        if not config:
            from ...config import config_obj
            config = config_obj
        # get from config/environment
        files_host = ENV_FILES_HOST.get(default=(config.get("api.files_server", ""))).rstrip('/')
        if files_host:
            return files_host

        # return default
        host = cls.get_api_server_host(config)
        if host == cls.default_host and cls.default_files:
            return cls.default_files

        # compose ourselves
        app_host = cls.get_app_server_host(config)
        parsed = urlparse(app_host)
        if parsed.port:
            parsed = parsed._replace(netloc=parsed.netloc.replace(':%d' % parsed.port, ':8081', 1))
        elif parsed.netloc.startswith('demoapp.'):
            parsed = parsed._replace(netloc=parsed.netloc.replace('demoapp.', 'demofiles.', 1))
        elif parsed.netloc.startswith('app.'):
            parsed = parsed._replace(netloc=parsed.netloc.replace('app.', 'files.', 1))
        else:
            parsed = parsed._replace(netloc=parsed.netloc + ':8081')

        return urlunparse(parsed) 

def quote_url(url):
    parsed = urlparse(url)
    if parsed.scheme not in ('http', 'https'):
        return url
    parsed = parsed._replace(path=quote(parsed.path))
    return urlunparse(parsed) 

def test_is_valid_url(url):
    unparsed_url = urllib_parse.urlunparse(url)
    assume(not unparsed_url.startswith("file://"))
    assert vistir.path.is_valid_url(unparsed_url)