Python six.moves.urllib.parse.urlparse() Examples

def test_error_view(self):
        client = app.test_client()

        auth_redirect = client.get('/')
        parsed_auth_request = dict(parse_qsl(urlparse(auth_redirect.location).query))

        # fake auth error response sent to redirect_uri
        error_auth_response = {
            'error': 'invalid_request',
            'error_description': 'test error',
            'state': parsed_auth_request['state']
        }
        error_page = client.get('/redirect_uri?{}'.format(urlencode(error_auth_response)), follow_redirects=True)

        assert json.loads(error_page.data.decode('utf-8')) == {
            'error': error_auth_response['error'],
            'message': error_auth_response['error_description']
        } 

def __init__(self, requirement_string):
        try:
            req = REQUIREMENT.parseString(requirement_string)
        except ParseException as e:
            raise InvalidRequirement(
                "Invalid requirement, parse error at \"{0!r}\"".format(
                    requirement_string[e.loc:e.loc + 8]))

        self.name = req.name
        if req.url:
            parsed_url = urlparse.urlparse(req.url)
            if not (parsed_url.scheme and parsed_url.netloc) or (
                    not parsed_url.scheme and not parsed_url.netloc):
                raise InvalidRequirement("Invalid URL given")
            self.url = req.url
        else:
            self.url = None
        self.extras = set(req.extras.asList() if req.extras else [])
        self.specifier = SpecifierSet(req.specifier)
        self.marker = req.marker if req.marker else None 

def router_login(base_url):
    """
    Prompt for router username and password and attempt login.

    Returns the username if successful or None.
    """
    config = PdtoolsConfig.load()
    session = requests.Session()
    url_parts = urlparse(base_url)

    for username, password in LoginGatherer(url_parts.netloc):
        # Try to get a token for later use. Prior to 1.10, paradrop-daemon
        # does not not support tokens.
        _, token = send_router_login(base_url, username, password, session)
        if token is not None:
            config.addAccessToken(url_parts.netloc, username, token)
            config.save()
            return username

    return None 

def send_router_login(request_url, username, password, session):
    url_parts = urlparse(request_url)

    auth_url = "{}://{}/api/v1/auth/local".format(url_parts.scheme, url_parts.netloc)
    data = {
        'username': username,
        'password': password
    }
    request = requests.Request('POST', auth_url, json=data)
    prepped = session.prepare_request(request)
    res = session.send(prepped)
    print("Server responded: {} {}".format(res.status_code, res.reason))

    if res.ok:
        data = res.json()
        token = data['token']

        return (res.status_code, token)

    else:
        return (res.status_code, None) 

def get_base_url(target):
    if target.startswith("http"):
        parts = urlparse(target)

        if parts.scheme != 'http':
            print("Warning: when specifying the Paradrop device address, "
                  "using a scheme ({}) other than http may result in errors."
                  .format(parts.scheme))

        if not parts.path:
            path = "/api/v1"
        else:
            print("Warning: when specifying the Paradrop device address, "
                  "using a path ({}) other than /api/v1 may result in errors."
                  .format(parts.scheme))
            path = parts.path

        return "{}://{}{}".format(parts.scheme, parts.netloc, path)

    else:
        return "http://{}/api/v1".format(target) 

def get_token(self):
        url_parts = urlparse(self.auth_url)

        print("Attempting to log in to authentication domain {}".format(url_parts.netloc))
        self.username = builtins.input("Username: ")
        password = getpass.getpass("Password: ")

        data = {
            self.param_map['username']: self.username,
            self.param_map['password']: password
        }
        res = requests.post(self.auth_url, json=data)
        try:
            data = res.json()
            self.token = data['token']
            return self.token
        except:
            return None 

def instance(cls, broker_id, zk_host, zk_port, zk_chroot, replicas,
                 partitions, message_max_bytes=1000000):
        if zk_chroot is None:
            zk_chroot = "afkak_" + str(uuid.uuid4()).replace("-", "_")
        if "KAFKA_URI" in os.environ:  # pragma: no cover
            parse = urlparse(os.environ["KAFKA_URI"])
            (host, port) = (parse.hostname, parse.port)
            fixture = ExternalService(host, port)
        else:
            (host, port) = ("127.0.0.1", get_open_port())
            fixture = cls(
                host=host, port=port, broker_id=broker_id, zk_host=zk_host,
                zk_port=zk_port, zk_chroot=zk_chroot, replicas=replicas,
                partitions=partitions, message_max_bytes=message_max_bytes,
            )
            fixture.open()
        return fixture 

def connect_device(uri):
    """
    Initialize device with uri, and set as current device.

    :param uri: an URI where to connect to device, e.g. `android://adbhost:adbport/serialno?param=value&param2=value2`
    :return: device instance
    :Example:
        * ``android:///`` # local adb device using default params
        * ``android://adbhost:adbport/1234566?cap_method=javacap&touch_method=adb``  # remote device using custom params
        * ``windows:///`` # local Windows application
        * ``ios:///`` # iOS device
    """
    d = urlparse(uri)
    platform = d.scheme
    host = d.netloc
    uuid = d.path.lstrip("/")
    params = dict(parse_qsl(d.query))
    if host:
        params["host"] = host.split(":")
    dev = init_device(platform, uuid, **params)
    return dev 

def test_default(self):
        target = TestTableauServerResultTarget(test_config.get_tmp_path('result.job'))
        target.datasource = 'test-datasource'
        url = urlparse(target.get_result_url())
        params = parse_qs(url.query)
        eq_(url.scheme, 'tableau')
        eq_(url.hostname, 'tableau.example.com')
        eq_(url.path, '/' + target.datasource)
        eq_(url_unquote(url.username), TestTableauServerResultTarget.username)
        eq_(url_unquote(url.password), TestTableauServerResultTarget.password)
        eq_(params.get('ssl'), ['true'])
        eq_(params.get('ssl_verify'), ['true'])
        eq_(params.get('server_version'), None)
        eq_(params.get('site'), None)
        eq_(params.get('project'), None)
        eq_(params.get('mode'), ['replace']) 

def test_get_relative_url_with_unicode(self):
        """Tests if it properly converts multibyte characters."""
        from six.moves.urllib import parse as urlparse

        self.view.request = self.request_factory.get(
            '/', data={'a': 1, 'b': 2}
        )
        expected_path = ('/elasticsearch/.kibana/search'
                         '/New-Saved-Search%E3%81%82')
        expected_qs = {'a': ['1'], 'b': ['2']}

        url = self.view.get_relative_url(
            u'/elasticsearch/.kibana/search/New-Saved-Searchあ'
        )
        # order of query params may change
        parsed_url = urlparse.urlparse(url)
        actual_path = parsed_url.path
        actual_qs = urlparse.parse_qs(parsed_url.query)

        self.assertEqual(actual_path, expected_path)
        self.assertEqual(actual_qs, expected_qs) 

def censor_connect_string(connect_string):
    """
    Take a SQLAlchemy connect string and return a sanitized version
    that can be written to the log without disclosing the password.
    The password is replaced with "xxxx".
    In case any error occurs, return "<error when censoring connect string>"
    """
    try:
        parsed = urlparse(connect_string)
        if parsed.password is not None:
            # We need to censor the ``netloc`` attribute: user:pass@host
            _, host = parsed.netloc.rsplit("@", 1)
            new_netloc = u'{}:{}@{}'.format(parsed.username, 'xxxx', host)
            # Convert the URL to six components. netloc is component #1.
            splitted = list(parsed)
            splitted[1] = new_netloc
            return urlunparse(splitted)
        return connect_string
    except Exception:
        return "<error when censoring connect string>" 

def scheme(self):
        return parse.urlparse(self.api._api_endpoint).scheme 

def combine_url(url, page, pagetitle, **kwargs):
    """ Add the specified arguments in the provided kwargs dictionary to
    the given URL.
    """
    url_obj = urlparse(url)
    url = url_obj.geturl().replace(url_obj.query, "").rstrip("?")
    query = {}
    for k, v in parse_qsl(url_obj.query):
        if k in query:
            if isinstance(query[k], list):
                query[k].append(v)
            else:
                query[k] = [query[k], v]
        else:
            query[k] = v
    query[pagetitle] = page
    query.update(kwargs)
    args = ""
    for key in query:
        if isinstance(query[key], list):
            for val in query[key]:
                args += "&%s=%s" % (key, val)
        else:
            args += "&%s=%s" % (key, query[key])
    return url + "?" + args[1:] 

def get_filename(self, task, default_ext):
        """Set the path where the image will be saved.

        The default strategy is to use an increasing 6-digit number as
        the filename. You can override this method if you want to set custom
        naming rules. The file extension is kept if it can be obtained from
        the url, otherwise ``default_ext`` is used as extension.

        Args:
            task (dict): The task dict got from ``task_queue``.

        Output:
            Filename with extension.
        """
        url_path = urlparse(task['file_url'])[2]
        extension = url_path.split('.')[-1] if '.' in url_path else default_ext
        file_idx = self.fetched_num + self.file_idx_offset
        return '{:06d}.{}'.format(file_idx, extension) 

def _decorate_request(self, filters, method, url, headers=None, body=None,
                          auth_data=None):
        if auth_data is None:
            auth_data = self.auth_data
        token, _ = auth_data
        base_url = self.base_url(filters=filters, auth_data=auth_data)
        # build authenticated request
        # returns new request, it does not touch the original values
        _headers = copy.deepcopy(headers) if headers is not None else {}
        _headers['X-Auth-Token'] = str(token)
        if url is None or url == "":
            _url = base_url
        else:
            # Join base URL and url, and remove multiple contiguous slashes
            _url = "/".join([base_url, url])
            parts = [x for x in urlparse.urlparse(_url)]
            parts[2] = re.sub("/{2,}", "/", parts[2])
            _url = urlparse.urlunparse(parts)
        # no change to method or body
        return str(_url), _headers, body 

def make_name_from_git(repo, branch, limit=53, separator='-', hash_size=5):
    """
    return name string representing the given git repo and branch
    to be used as a build name.

    NOTE: Build name will be used to generate pods which have a
    limit of 64 characters and is composed as:

        <buildname>-<buildnumber>-<podsuffix>
        rhel7-1-build

    Assuming '-XXXX' (5 chars) and '-build' (6 chars) as default
    suffixes, name should be limited to 53 chars (64 - 11).

    OpenShift is very peculiar in which BuildConfig names it
    allows. For this reason, only certain characters are allowed.
    Any disallowed characters will be removed from repo and
    branch names.

    :param repo: str, the git repository to be used
    :param branch: str, the git branch to be used
    :param limit: int, max name length
    :param separator: str, used to separate the repo and branch in name

    :return: str, name representing git repo and branch.
    """

    branch = branch or 'unknown'
    full = urlparse(repo).path.lstrip('/') + branch
    repo = git_repo_humanish_part_from_uri(repo)
    shaval = sha256(full.encode('utf-8')).hexdigest()
    hash_str = shaval[:hash_size]
    limit = limit - len(hash_str) - 1

    sanitized = sanitize_strings_for_openshift(repo, branch, limit, separator, False)
    return separator.join(filter(None, (sanitized, hash_str))) 

def anon_login(self, folder_token, folder_password=''):
        self._session = requests.session()
        self._session.verify = self._verify_certs
        self._session.auth = (folder_token, folder_password)

        url_components = parse.urlparse(self.url)
        self._davpath = url_components.path + 'public.php/webdav'
        self._webdav_url = self.url + 'public.php/webdav' 

def login(self, user_id, password):
        """Authenticate to ownCloud.
        This will create a session on the server.

        :param user_id: user id
        :param password: password
        :raises: HTTPResponseError in case an HTTP error status was returned
        """

        self._session = requests.session()
        self._session.verify = self._verify_certs
        self._session.auth = (user_id, password)

        try:
            self._update_capabilities()

            url_components = parse.urlparse(self.url)
            if self._dav_endpoint_version == 1:
                self._davpath = url_components.path + 'remote.php/dav/files/' + parse.quote(user_id)
                self._webdav_url = self.url + 'remote.php/dav/files/' + parse.quote(user_id)
            else:
                self._davpath = url_components.path + 'remote.php/webdav'
                self._webdav_url = self.url + 'remote.php/webdav'

        except HTTPResponseError as e:
            self._session.close()
            self._session = None
            raise e 

def do_GET(self):  # pylint: disable=invalid-name
    """Handler for all get requests."""
    parsed_url = urlparse.urlparse(self.path)

    # Remove a trailing slash, if present.
    clean_path = parsed_url.path
    if clean_path.endswith('/'):
      clean_path = clean_path[:-1]

    query_params = urlparse.parse_qs(parsed_url.query)
    # parse_qs returns a list of values for each key; we're only interested in
    # the first.
    for key in query_params:
      value_count = len(query_params[key])
      if value_count != 1:
        self.respond(
            'query parameter %s should have exactly one value, had %d' %
            (key, value_count), 'text/plain', 400)
        return
      query_params[key] = query_params[key][0]

    if clean_path in self.data_handlers:
      self.data_handlers[clean_path](query_params)
    elif clean_path in TAB_ROUTES:
      self._serve_index(query_params)
    else:
      self._serve_static_file(clean_path)

  # @Override 

def move_to_destination(source, destination, job_name, sagemaker_session):
    """move source to destination. Can handle uploading to S3

    Args:
        source (str): root directory to move
        destination (str): file:// or s3:// URI that source will be moved to.
        job_name (str): SageMaker job name.
        sagemaker_session (sagemaker.Session): a sagemaker_session to interact
            with S3 if needed

    Returns:
        (str): destination URI
    """
    parsed_uri = urlparse(destination)
    if parsed_uri.scheme == "file":
        recursive_copy(source, parsed_uri.path)
        final_uri = destination
    elif parsed_uri.scheme == "s3":
        bucket = parsed_uri.netloc
        path = _create_s3_prefix(parsed_uri.path, job_name)
        final_uri = "s3://%s/%s" % (bucket, path)
        sagemaker_session.upload_data(source, bucket, path)
    else:
        raise ValueError("Invalid destination URI, must be s3:// or file://, got: %s" % destination)

    shutil.rmtree(source)
    return final_uri 

def serve(self, model_dir, environment):
        """Host a local endpoint using docker-compose.
        Args:
            primary_container (dict): dictionary containing the container runtime settings
                for serving. Expected keys:
                - 'ModelDataUrl' pointing to a file or s3:// location.
                - 'Environment' a dictionary of environment variables to be passed to the
                    hosting container.

        """
        logger.info("serving")

        self.container_root = self._create_tmp_folder()
        logger.info("creating hosting dir in %s", self.container_root)

        volumes = self._prepare_serving_volumes(model_dir)

        # If the user script was passed as a file:// mount it to the container.
        if sagemaker.estimator.DIR_PARAM_NAME.upper() in environment:
            script_dir = environment[sagemaker.estimator.DIR_PARAM_NAME.upper()]
            parsed_uri = urlparse(script_dir)
            if parsed_uri.scheme == "file":
                volumes.append(_Volume(parsed_uri.path, "/opt/ml/code"))
                # Update path to mount location
                environment = environment.copy()
                environment[sagemaker.estimator.DIR_PARAM_NAME.upper()] = "/opt/ml/code"

        if _ecr_login_if_needed(self.sagemaker_session.boto_session, self.image):
            _pull_image(self.image)

        self._generate_compose_file(
            "serve", additional_env_vars=environment, additional_volumes=volumes
        )
        compose_command = self._compose()

        self.container = _HostingContainer(compose_command)
        self.container.start() 

def ensure_rpm(state, host, files, source, present, package_manager_command):
    # If source is a url
    if urlparse(source).scheme:
        # Generate a temp filename (with .rpm extension to please yum)
        temp_filename = '{0}.rpm'.format(state.get_temp_filename(source))

        # Ensure it's downloaded
        yield files.download(state, host, source, temp_filename)

        # Override the source with the downloaded file
        source = temp_filename

    # Check for file .rpm information
    info = host.fact.rpm_package(source)
    exists = False

    # We have info!
    if info:
        current_package = host.fact.rpm_package(info['name'])
        if current_package and current_package['version'] == info['version']:
            exists = True

    # Package does not exist and we want?
    if present and not exists:
        # If we had info, always install
        if info:
            yield 'rpm -i {0}'.format(source)

        # This happens if we download the package mid-deploy, so we have no info
        # but also don't know if it's installed. So check at runtime, otherwise
        # the install will fail.
        else:
            yield 'rpm -q `rpm -qp {0}` 2> /dev/null || rpm -i {0}'.format(source)

    # Package exists but we don't want?
    if exists and not present:
        yield '{0} remove -y {1}'.format(package_manager_command, info['name']) 

def test_prepare_preflight_url(self):
        """
        Check if preflight request is properly formed and has all required keys.
        """
        preflight_request_data = self.lti_consumer.prepare_preflight_url(
            callback_url=LAUNCH_URL,
            hint="test-hint",
            lti_hint="test-lti-hint"
        )

        # Extract and check parameters from OIDC launch request url
        parameters = parse_qs(urlparse(preflight_request_data['oidc_url']).query)
        self.assertCountEqual(
            parameters.keys(),
            [
                'iss',
                'login_hint',
                'lti_message_hint',
                'client_id',
                'target_link_uri',
                'lti_deployment_id'
            ]
        )
        self.assertEqual(parameters['iss'][0], ISS)
        self.assertEqual(parameters['client_id'][0], CLIENT_ID)
        self.assertEqual(parameters['login_hint'][0], "test-hint")
        self.assertEqual(parameters['lti_message_hint'][0], "test-lti-hint")
        self.assertEqual(parameters['lti_deployment_id'][0], DEPLOYMENT_ID)
        self.assertEqual(parameters['target_link_uri'][0], LAUNCH_URL) 

def key(state, host, key=None, keyserver=None, keyid=None):
    '''
    Add apt gpg keys with ``apt-key``.

    + key: filename or URL
    + keyserver: URL of keyserver to fetch key from
    + keyid: key identifier when using keyserver

    Note:
        Always returns an add command, not state checking.

    keyserver/id:
        These must be provided together.

    Examples:

    .. code:: python

        # Note: If using URL, wget is assumed to be installed.
        apt.key(
            {'Add the Docker apt gpg key'},
            key='https://download.docker.com/linux/ubuntu/gpg',
        )

        apt.key(
            {'Install VirtualBox key'},
            'https://www.virtualbox.org/download/oracle_vbox_2016.asc',
        )

    '''

    if key:
        # If URL, wget the key to stdout and pipe into apt-key, because the "adv"
        # apt-key passes to gpg which doesn't always support https!
        if urlparse(key).scheme:
            yield 'wget -O- {0} | apt-key add -'.format(key)
        else:
            yield 'apt-key add {0}'.format(key)

    if keyserver and keyid:
        yield 'apt-key adv --keyserver {0} --recv-keys {1}'.format(keyserver, keyid) 

def is_url(s):
    """
    Check if the argument is an URL.

    Returns:
        True if the argument is an URL.
    """
    return urlparse(s).scheme in ('http', 'https') 

def __call__(self, environ, start_response):  # pylint: disable=invalid-name
    """Central entry point for the TensorBoard application.

    This method handles routing to sub-applications. It does simple routing
    using regular expression matching.

    This __call__ method conforms to the WSGI spec, so that instances of this
    class are WSGI applications.

    Args:
      environ: See WSGI spec.
      start_response: See WSGI spec.

    Returns:
      A werkzeug Response.
    """
    request = wrappers.Request(environ)
    parsed_url = urlparse.urlparse(request.path)

    # Remove a trailing slash, if present.
    clean_path = parsed_url.path
    if clean_path.endswith('/'):
      clean_path = clean_path[:-1]
    # pylint: disable=too-many-function-args
    if clean_path in self.data_applications:
      return self.data_applications[clean_path](environ, start_response)
    elif clean_path in TAB_ROUTES:
      return self._serve_index(environ, start_response)
    else:
      return self._serve_static_file(request, clean_path)(environ,
                                                          start_response)
    # pylint: enable=too-many-function-args 

def __init__(self, requirement_string):
        try:
            req = REQUIREMENT.parseString(requirement_string)
        except ParseException as e:
            raise InvalidRequirement(
                'Parse error at "{0!r}": {1}'.format(
                    requirement_string[e.loc : e.loc + 8], e.msg
                )
            )

        self.name = req.name
        if req.url:
            parsed_url = urlparse.urlparse(req.url)
            if parsed_url.scheme == "file":
                if urlparse.urlunparse(parsed_url) != req.url:
                    raise InvalidRequirement("Invalid URL given")
            elif not (parsed_url.scheme and parsed_url.netloc) or (
                not parsed_url.scheme and not parsed_url.netloc
            ):
                raise InvalidRequirement("Invalid URL: {0}".format(req.url))
            self.url = req.url
        else:
            self.url = None
        self.extras = set(req.extras.asList() if req.extras else [])
        self.specifier = SpecifierSet(req.specifier)
        self.marker = req.marker if req.marker else None 

def __init__(self, requirement_string):
        try:
            req = REQUIREMENT.parseString(requirement_string)
        except ParseException as e:
            raise InvalidRequirement(
                'Parse error at "{0!r}": {1}'.format(
                    requirement_string[e.loc : e.loc + 8], e.msg
                )
            )

        self.name = req.name
        if req.url:
            parsed_url = urlparse.urlparse(req.url)
            if parsed_url.scheme == "file":
                if urlparse.urlunparse(parsed_url) != req.url:
                    raise InvalidRequirement("Invalid URL given")
            elif not (parsed_url.scheme and parsed_url.netloc) or (
                not parsed_url.scheme and not parsed_url.netloc
            ):
                raise InvalidRequirement("Invalid URL: {0}".format(req.url))
            self.url = req.url
        else:
            self.url = None
        self.extras = set(req.extras.asList() if req.extras else [])
        self.specifier = SpecifierSet(req.specifier)
        self.marker = req.marker if req.marker else None 

def is_schema_url(path):
    result = urlparse(path)
    try:
        return all([result.scheme, result.netloc, result.path])
    except AttributeError:
        return False 

def _parse_url(db_url):
    """parse database connection urls into components

    UTA database connection URLs follow that of SQLAlchemy, except
    that a schema may be optionally specified after the database. The
    skeleton format is:

       driver://user:pass@host/database/schema

    >>> params = _parse_url("driver://user:pass@host:9876/database/schema")

    >>> params.scheme
    u'driver'

    >>> params.hostname
    u'host'

    >>> params.username
    u'user'

    >>> params.password
    u'pass'

    >>> params.database
    u'database'

    >>> params.schema
    u'schema'

    """

    return ParseResult(urlparse.urlparse(db_url))