Python six.moves.urllib.parse.urlsplit() Examples

def add_repo(repo_name=None, repo_url=None):
    """Adds a new repo to the system based on the repo_url"""
    if repo_url is not None:
        if repo_name is None:
            repo_name = urlsplit(repo_url).netloc
        repo_file = (
            '[{0}]\n'
            'name={0}\n'
            'baseurl={1}\n'
            'enabled=1\n'.format(repo_name, repo_url)
        )
        with cd('/etc/yum.repos.d/'):
            run('echo "{0}" >> automation-tools.repo'.format(repo_file))
    else:
        print('add_repo requires a repo_url to make any changes.')
        sys.exit(1)


# Client registration
# ================================================== 

def update_query_parameters(url, query_parameters):
    """
    Return url with updated query parameters.

    Arguments:
        url (str): Original url whose query parameters need to be updated.
        query_parameters (dict): A dictionary containing query parameters to be added to course selection url.

    Returns:
        (slug): slug identifier for the identity provider that can be used for identity verification of
            users associated the enterprise customer of the given user.

    """
    scheme, netloc, path, query_string, fragment = urlsplit(url)
    url_params = parse_qs(query_string)

    # Update url query parameters
    url_params.update(query_parameters)

    return urlunsplit(
        (scheme, netloc, path, urlencode(sorted(url_params.items()), doseq=True), fragment),
    ) 

def test_logged_out_wasapi_list_basic_auth(self):
        self.testapp.authorization = ('Basic', ('someuser', 'Password1'))
        res = self.testapp.get('/api/v1/download/webdata')
        self.testapp.authorization = None

        assert len(res.json['files']) == 3
        assert res.json['files'][0]['checksums']
        assert res.json['files'][0]['locations']

        assert sum((1 if val['is_active'] else 0 for val in res.json['files']), 0) == 2

        wasapi_filename = res.json['files'][0]['locations'][0]

        # 404 without basic auth
        res = self.testapp.head(urlsplit(wasapi_filename).path, status=404)

        # 200 with basic auth
        self.testapp.authorization = ('Basic', ('someuser', 'Password1'))
        res = self.testapp.head(urlsplit(wasapi_filename).path)
        assert res.headers['Content-Disposition'].startswith("attachment; filename*=UTF-8''rec-")
        self.testapp.authorization = None 

def get_commit_url(commit, pkg):
    try:
        upstream_url = parse.urlsplit(pkg["upstream"])

        if upstream_url.netloc == "git.openstack.org":
            commit_url = ("http",
                          upstream_url.netloc,
                          "/cgit%s/commit/?id=" % upstream_url.path,
                          "", "", "")
            commit_url = parse.urlunparse(commit_url)
        elif upstream_url.netloc == "github.com":
            commit_url = ("https",
                          upstream_url.netloc,
                          "%s/commit/" % upstream_url.path,
                          "", "", "")
            commit_url = parse.urlunparse(commit_url)
        else:
            # Fallback when no cgit URL can be defined
            commit_url = pkg["upstream"]
    except KeyError:
        # This should not happen, but pkg['upstream'] may not be present
        # after some error in the gitrepo driver
        commit_url = ''

    return commit_url 

def test_notify_alarm(self):
        data = {
            'actions': ['test://'],
            'alarm_id': 'foobar',
            'alarm_name': 'testalarm',
            'severity': 'critical',
            'previous': 'OK',
            'current': 'ALARM',
            'reason': 'Everything is on fire',
            'reason_data': {'fire': 'everywhere'}
        }
        self._msg_notifier.sample({}, 'alarm.update', data)
        time.sleep(1)
        notifications = self.service.notifiers['test'].obj.notifications
        self.assertEqual(1, len(notifications))
        self.assertEqual((urlparse.urlsplit(data['actions'][0]),
                          data['alarm_id'],
                          data['alarm_name'],
                          data['severity'],
                          data['previous'],
                          data['current'],
                          data['reason'],
                          data['reason_data']),
                         notifications[0]) 

def get_current_ver(cls, election_url):
        election_url_parts = parse.urlsplit(cls._url_ensure_trailing_slash(election_url))
        if 'Web02' in election_url:
            cls.parsed_url = election_url_parts._replace(path="/".join(election_url_parts.path.split('/')[:3]) + "/current_ver.txt", fragment='')
            election_url_parts =  cls.parsed_url
        else:
            election_url_parts = election_url_parts._replace(path=election_url_parts.path + "current_ver.txt")

        current_ver_url = parse.urlunsplit(election_url_parts)

        current_ver_response = requests.get(current_ver_url)

        try:
            current_ver_response.raise_for_status()
        except requests.exceptions.HTTPError:
            return None

        return current_ver_response.text 

def _test_warc_write(self, url, user, coll, rec):
        parts = urlsplit(url)
        host = parts.netloc
        path = parts.path
        if parts.query:
            path += '?' + parts.query

        # Rec must be open
        #self.redis.hset('r:{user}:{coll}:{rec}:info'.format(user=user, coll=coll, rec=rec), 'id', rec)
        self.redis.setex('r:{rec}:open'.format(user=user, coll=coll, rec=rec), 30, 1)
        self.redis.hset('u:{user}:info'.format(user=user), 'size', 0)
        self.redis.hset('u:{user}:info'.format(user=user), 'max_size', 10000)

        req_url = '/record/live/resource/postreq?url={url}&param.recorder.user={user}&param.recorder.coll={coll}&param.recorder.rec={rec}'
        req_url = req_url.format(url=quote(url), user=user, coll=coll, rec=rec)
        resp = self.testapp.post(req_url, general_req_data.format(host=host, path=path).encode('utf-8'))

        while not self.wr_rec.recorder.write_queue.empty():
            self.wr_rec.recorder._write_one()

        assert resp.headers['Warcserver-Source-Coll'] == 'live'

        return resp 

def _get_session(self, url):
        if self._connection_pool:
            magic_tuple = parse.urlsplit(url)
            scheme, netloc, path, query, frag = magic_tuple
            service_url = '%s://%s' % (scheme, netloc)
            if self._current_url != service_url:
                # Invalidate Session object in case the url is somehow changed
                if self._session:
                    self._session.close()
                self._current_url = service_url
                self._logger.debug(
                    "New session created for: (%s)" % service_url)
                self._session = requests.Session()
                self._session.mount(service_url,
                                    self._connection_pool.get(service_url))
            return self._session
        elif self._session:
            return self._session 

def test_anon_record_redirect_and_delete(self):
        self.set_uuids('Recording', ['recording-session'])
        res = self.testapp.get('/record/mp_/http://example.com/')
        assert res.status_code == 302

        parts = urlsplit(res.headers['Location'])

        path_parts = parts.path.split('/', 2)
        assert self.anon_user == path_parts[1]

        assert self.anon_user.startswith(Session.temp_prefix)
        assert parts.path.endswith('/temp/recording-session/record/mp_/http://example.com/')

        # Delete this recording
        res = self.testapp.delete('/api/v1/recording/recording-session?user={user}&coll=temp'.format(user=self.anon_user))

        assert res.json == {'deleted_id': 'recording-session'}


        assert int(self.redis.hget(User.INFO_KEY.format(user=self.anon_user), Stats.DELETE_PROP)) > 0 

def _get_session(self, url):
        if self._connection_pool:
            magic_tuple = parse.urlsplit(url)
            scheme, netloc, path, query, frag = magic_tuple
            service_url = '%s://%s' % (scheme, netloc)
            if self._current_url != service_url:
                # Invalidate Session object in case the url is somehow changed
                if self._session:
                    self._session.close()
                self._current_url = service_url
                self._logger.debug(
                    "New session created for: (%s)" % service_url)
                self._session = requests.Session()
                self._session.mount(service_url,
                                    self._connection_pool.get(service_url))
            return self._session
        elif self._session:
            return self._session

    # @set_headers_param 

def load_tests(loader, tests, pattern):
    """Provide a TestSuite to the discovery process."""
    aodh_url = os.getenv('AODH_URL')
    if aodh_url:
        parsed_url = urlparse.urlsplit(aodh_url)
        prefix = parsed_url.path.rstrip('/')  # turn it into a prefix

        # NOTE(chdent): gabbi requires a port be passed or it will
        # default to 8001, so we must dance a little dance to get
        # the right ports. Probably gabbi needs to change.
        # https://github.com/cdent/gabbi/issues/50
        port = 443 if parsed_url.scheme == 'https' else 80
        if parsed_url.port:
            port = parsed_url.port

        test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
        return driver.build_tests(test_dir, loader,
                                  host=parsed_url.hostname,
                                  port=port,
                                  prefix=prefix)
    elif os.getenv('GABBI_LIVE_FAIL_IF_NO_TEST'):
        raise RuntimeError('AODH_URL is not set') 

def _get_session(self, url):
        if self._connection_pool:
            magic_tuple = parse.urlsplit(url)
            scheme, netloc, path, query, frag = magic_tuple
            service_url = '%s://%s' % (scheme, netloc)
            if self._current_url != service_url:
                # Invalidate Session object in case the url is somehow changed
                if self._session:
                    self._session.close()
                self._current_url = service_url
                self._logger.debug(
                    "New session created for: (%s)" % service_url)
                self._session = requests.Session()
                self._session.mount(service_url,
                                    self._connection_pool.get(service_url))
            return self._session
        elif self._session:
            return self._session 

def _bucket_bound_hostname_url(host, scheme=None):
    """Helper to build bucket bound hostname URL.

    :type host: str
    :param host: Host name.

    :type scheme: str
    :param scheme: (Optional) Web scheme. If passed, use it
                   as a scheme in the result URL.

    :rtype: str
    :returns: A bucket bound hostname URL.
    """
    url_parts = urlsplit(host)
    if url_parts.scheme and url_parts.netloc:
        return host

    return "{scheme}://{host}/".format(scheme=scheme, host=host) 

def remove_trailing_version_from_href(href):
    """Removes the api version from the href.

    Given: 'http://www.masakari.com/ha/v1.1'
    Returns: 'http://www.masakari.com/ha'

    Given: 'http://www.masakari.com/v1.1'
    Returns: 'http://www.masakari.com'

    """
    parsed_url = urlparse.urlsplit(href)
    url_parts = parsed_url.path.rsplit('/', 1)

    # NOTE: this should match vX.X or vX
    expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)')
    if not expression.match(url_parts.pop()):
        LOG.debug('href %s does not contain version', href)
        raise ValueError(_('href %s does not contain version') % href)

    new_path = url_join(*url_parts)
    parsed_url = list(parsed_url)
    parsed_url[2] = new_path
    return urlparse.urlunsplit(parsed_url) 

def _add_query_parameters(base_url, name_value_pairs):
    """Add one query parameter to a base URL.

    :type base_url: string
    :param base_url: Base URL (may already contain query parameters)

    :type name_value_pairs: list of (string, string) tuples.
    :param name_value_pairs: Names and values of the query parameters to add

    :rtype: string
    :returns: URL with additional query strings appended.
    """
    if len(name_value_pairs) == 0:
        return base_url

    scheme, netloc, path, query, frag = urlsplit(base_url)
    query = parse_qsl(query)
    query.extend(name_value_pairs)
    return urlunsplit((scheme, netloc, path, urlencode(query), frag)) 

def urlsplit(url, scheme='', allow_fragments=True):
    """Parse a URL using urlparse.urlsplit(), splitting query and fragments.
    This function papers over Python issue9374_ when needed.

    .. _issue9374: http://bugs.python.org/issue9374

    The parameters are the same as urlparse.urlsplit.
    """
    scheme, netloc, path, query, fragment = parse.urlsplit(
        url, scheme, allow_fragments)
    if allow_fragments and '#' in path:
        path, fragment = path.split('#', 1)
    if '?' in path:
        path, query = path.split('?', 1)
    return _ModifiedSplitResult(scheme, netloc,
                                path, query, fragment) 

def append_warc(self, dest, path):
        if not path.startswith('s3://'):
            logging.info('SKIPPING INVALID ' + path)
            return

        parts = urlsplit(path)

        src_bucket = self.s3conn.get_bucket(parts.netloc)

        stream = src_bucket.get_key(parts.path)

        stream.open_read()

        shutil.copyfileobj(stream, dest)

    # from BaseController 

def load_tests(loader, tests, pattern):
    """Provide a TestSuite to the discovery process."""
    gnocchi_url = os.getenv('GNOCCHI_ENDPOINT')
    if gnocchi_url:
        parsed_url = urlparse.urlsplit(gnocchi_url)
        prefix = parsed_url.path.rstrip('/')  # turn it into a prefix

        # NOTE(chdent): gabbi requires a port be passed or it will
        # default to 8001, so we must dance a little dance to get
        # the right ports. Probably gabbi needs to change.
        # https://github.com/cdent/gabbi/issues/50
        port = 443 if parsed_url.scheme == 'https' else 80
        if parsed_url.port:
            port = parsed_url.port

        test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
        return driver.build_tests(test_dir, loader,
                                  host=parsed_url.hostname,
                                  port=port,
                                  prefix=prefix)
    elif os.getenv("GABBI_LIVE"):
        raise RuntimeError('"GNOCCHI_ENDPOINT" is not set') 

def _build_url(self, url):
        """Build a complete URL from a path by substituting in session parameters."""
        components = urlsplit(url)

        domain = components.netloc or self._settings['domain']

        # Add port if a non-standard port was specified
        if self._settings['port'] is not None and ':' not in domain:
            domain = '{}:{}'.format(domain, self._settings['port'])

        return urlunsplit([
            components.scheme or self._settings['protocol'],
            domain,
            components.path,
            components.query,
            components.fragment
        ]) 

def _get_url_base(cls, url):
        """
        Return the base of a URL
        """
        s = urlsplit(url)
        return urlunsplit([s.scheme, s.netloc, '', '', '']) 

def photo_is_local(self, request_url):
        # TODO(nworden): consider setting the acceptable domain in
        # site_settings.py, so that we don't have to pass a request URL in. It's
        # not obvious how to account for different App Engine versions without
        # it being a hassle, but shouldn't be too hard.
        if not self.photo_url:
            return False
        else:
            _, our_netloc, _, _, _ = urlparse.urlsplit(request_url)
            _, photo_netloc, _, _, _ = urlparse.urlsplit(self.photo_url)
            return photo_netloc == our_netloc 

def hostname_for_event(self, clean_server_name):
        """Return a reasonable hostname for a replset membership event to mention."""
        uri = urlsplit(clean_server_name)
        if '@' in uri.netloc:
            hostname = uri.netloc.split('@')[1].split(':')[0]
        else:
            hostname = uri.netloc.split(':')[0]
        if hostname == 'localhost':
            hostname = self.hostname
        return hostname 

def from_path(cls, path):
        """Takes a path and returns a File object with the path as the PFN."""
        urlparts = urlsplit(path)
        site = 'nonlocal'
        if (urlparts.scheme == '' or urlparts.scheme == 'file'):
            if os.path.isfile(urlparts.path):
                path = os.path.abspath(urlparts.path)
                path = urljoin('file:', pathname2url(path)) 
                site = 'local'

        fil = File(os.path.basename(path))
        fil.PFN(path, site)
        return fil 

def payment_domain_name(self):
        if self.enable_microfrontend_for_basket_page:
            return urlsplit(self.payment_microfrontend_url).netloc
        return self.site.domain 

def assertUrlEqual(self, expected_url, provided_url):

        def parse_components(url):
            parsed = parse.urlsplit(url)
            query = parse.parse_qs(parsed.query)
            return parsed._replace(query=''), query

        expected_parse, expected_query = parse_components(expected_url)
        provided_parse, provided_query = parse_components(provided_url)

        self.assertEqual(expected_parse, provided_parse)
        self.assertEqual(expected_query, provided_query) 

def _get_url_parts(url):
    url = _clean_url(url)
    return urlsplit(url) 

def get_base_url_from_request():
    """Derive base url from wsgi request if CONF.host_href is not set

    Use host.href as base URL if its set in barbican.conf.
    If its not set, then derives value from wsgi request. WSGI request uses
    HOST header or HTTP_X_FORWARDED_FOR header (in case of proxy) for host +
    port part of its url. Proxies can also set HTTP_X_FORWARDED_PROTO header
    for indicating http vs https.

    Some of unit tests does not have pecan context that's why using request
    attr check on pecan instance.
    """
    if not CONF.host_href and hasattr(pecan.request, 'application_url'):
        p_url = parse.urlsplit(pecan.request.application_url)
        # Pecan does not handle X_FORWARDED_PROTO yet, so we need to
        # handle it ourselves. see lp#1445290
        scheme = pecan.request.environ.get('HTTP_X_FORWARDED_PROTO', 'http')
        # Pecan does not handle url reconstruction according to
        # https://www.python.org/dev/peps/pep-0333/#url-reconstruction
        netloc = pecan.request.environ.get('HTTP_HOST', p_url.netloc)
        # FIXME: implement SERVER_NAME lookup if HTTP_HOST is not set
        if p_url.path:
            # Remove the version from the path to extract the base path
            base_path = re.sub('/v[0-9\.]+$', '', p_url.path)
            base_url = '%s://%s%s' % (scheme, netloc, base_path)
        else:
            base_url = '%s://%s' % (scheme, netloc)
        return base_url
    else:  # when host_href is set or flow is not within wsgi request context
        return CONF.host_href 

def test_wasapi_list(self):
        res = self.testapp.get('/api/v1/download/webdata')
        assert len(res.json['files']) == 2
        assert res.json['files'][0]['checksums']
        assert res.json['files'][0]['locations']

        assert res.json['files'][1]['checksums']
        assert res.json['files'][1]['locations']

        assert sum((1 if val['is_active'] else 0 for val in res.json['files']), 0) == 1

        wasapi_filename = res.json['files'][0]['locations'][0]
        res = self.testapp.head(urlsplit(wasapi_filename).path)

        assert res.headers['Content-Disposition'].startswith("attachment; filename*=UTF-8''rec-") 

def __init__(self, src_redis_url, dst_redis_url, dest_prefix, dry_run=True):
        self.src_redis = redis.StrictRedis.from_url(src_redis_url, decode_responses=True)
        self.dst_redis = redis.StrictRedis.from_url(dst_redis_url, decode_responses=True)
        self.dry = dry_run

        self.s3conn = boto.connect_s3()

        parts = urlsplit(dest_prefix)
        self.dest_bucket_name = parts.netloc
        self.path_prefix = parts.path[1:]

        self.dest_bucket = self.s3conn.get_bucket(self.dest_bucket_name) 

def _split_bucket_path(self, url):
        """Split S3 bucket URL into network location and path.

        :param str url: S3 bucket URL

        :returns: network location and path
        :rtype: str and str
        """
        parts = urlsplit(url)
        return parts.netloc, parts.path.lstrip('/')