Python six.moves.urllib.parse() Examples

def find_external_links(url, page):
    """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""

    for match in REL.finditer(page):
        tag, rel = match.groups()
        rels = set(map(str.strip, rel.lower().split(',')))
        if 'homepage' in rels or 'download' in rels:
            for match in HREF.finditer(tag):
                yield urllib.parse.urljoin(url, htmldecode(match.group(1)))

    for tag in ("<th>Home Page", "<th>Download URL"):
        pos = page.find(tag)
        if pos != -1:
            match = HREF.search(page, pos)
            if match:
                yield urllib.parse.urljoin(url, htmldecode(match.group(1))) 

def find_external_links(url, page):
    """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""

    for match in REL.finditer(page):
        tag, rel = match.groups()
        rels = set(map(str.strip, rel.lower().split(',')))
        if 'homepage' in rels or 'download' in rels:
            for match in HREF.finditer(tag):
                yield urllib.parse.urljoin(url, htmldecode(match.group(1)))

    for tag in ("<th>Home Page", "<th>Download URL"):
        pos = page.find(tag)
        if pos != -1:
            match = HREF.search(page, pos)
            if match:
                yield urllib.parse.urljoin(url, htmldecode(match.group(1))) 

def _download_svn(self, url, filename):
        url = url.split('#', 1)[0]  # remove any fragment for svn's sake
        creds = ''
        if url.lower().startswith('svn:') and '@' in url:
            scheme, netloc, path, p, q, f = urllib.parse.urlparse(url)
            if not netloc and path.startswith('//') and '/' in path[2:]:
                netloc, path = path[2:].split('/', 1)
                auth, host = splituser(netloc)
                if auth:
                    if ':' in auth:
                        user, pw = auth.split(':', 1)
                        creds = " --username=%s --password=%s" % (user, pw)
                    else:
                        creds = " --username=" + auth
                    netloc = host
                    parts = scheme, netloc, url, p, q, f
                    url = urllib.parse.urlunparse(parts)
        self.info("Doing subversion checkout from %s to %s", url, filename)
        os.system("svn checkout%s -q %s %s" % (creds, url, filename))
        return filename 

def _http_request(url, headers=None, data=None, method="GET", timeout=None):
    import socket
    from six.moves import urllib

    headers = headers or {}
    url_parts = urllib.parse.urlparse(url)
    conn = _HTTPConnection(url_parts.scheme, url_parts.netloc, timeout)
    params = urllib.parse.urlencode(data) if data else ""
    try:
        conn.request(method, url_parts.path, params, headers)
    except socket.error as e:
        if e.errno == errno.ECONNREFUSED:
            raise HTTPConnectionError(url)
        raise
    else:
        return HTTPResponse(conn.getresponse()) 

def _vcs_split_rev_from_url(url, pop_prefix=False):
        scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)

        scheme = scheme.split('+', 1)[-1]

        # Some fragment identification fails
        path = path.split('#', 1)[0]

        rev = None
        if '@' in path:
            path, rev = path.rsplit('@', 1)

        # Also, discard fragment
        url = urllib.parse.urlunsplit((scheme, netloc, path, query, ''))

        return url, rev 

def _encode_auth(auth):
    """
    A function compatible with Python 2.3-3.3 that will encode
    auth from a URL suitable for an HTTP header.
    >>> str(_encode_auth('username%3Apassword'))
    'dXNlcm5hbWU6cGFzc3dvcmQ='

    Long auth strings should not cause a newline to be inserted.
    >>> long_auth = 'username:' + 'password'*10
    >>> chr(10) in str(_encode_auth(long_auth))
    False
    """
    auth_s = urllib.parse.unquote(auth)
    # convert to bytes
    auth_bytes = auth_s.encode()
    # use the legacy interface for Python 2.3 support
    encoded_bytes = base64.encodestring(auth_bytes)
    # convert back to a string
    encoded = encoded_bytes.decode()
    # strip the trailing carriage return
    return encoded.replace('\n', '') 

def _download_svn(self, url, filename):
        url = url.split('#', 1)[0]  # remove any fragment for svn's sake
        creds = ''
        if url.lower().startswith('svn:') and '@' in url:
            scheme, netloc, path, p, q, f = urllib.parse.urlparse(url)
            if not netloc and path.startswith('//') and '/' in path[2:]:
                netloc, path = path[2:].split('/', 1)
                auth, host = splituser(netloc)
                if auth:
                    if ':' in auth:
                        user, pw = auth.split(':', 1)
                        creds = " --username=%s --password=%s" % (user, pw)
                    else:
                        creds = " --username=" + auth
                    netloc = host
                    parts = scheme, netloc, url, p, q, f
                    url = urllib.parse.urlunparse(parts)
        self.info("Doing subversion checkout from %s to %s", url, filename)
        os.system("svn checkout%s -q %s %s" % (creds, url, filename))
        return filename 

def _vcs_split_rev_from_url(url, pop_prefix=False):
        scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)

        scheme = scheme.split('+', 1)[-1]

        # Some fragment identification fails
        path = path.split('#', 1)[0]

        rev = None
        if '@' in path:
            path, rev = path.rsplit('@', 1)

        # Also, discard fragment
        url = urllib.parse.urlunsplit((scheme, netloc, path, query, ''))

        return url, rev 

def _encode_auth(auth):
    """
    A function compatible with Python 2.3-3.3 that will encode
    auth from a URL suitable for an HTTP header.
    >>> str(_encode_auth('username%3Apassword'))
    'dXNlcm5hbWU6cGFzc3dvcmQ='

    Long auth strings should not cause a newline to be inserted.
    >>> long_auth = 'username:' + 'password'*10
    >>> chr(10) in str(_encode_auth(long_auth))
    False
    """
    auth_s = urllib.parse.unquote(auth)
    # convert to bytes
    auth_bytes = auth_s.encode()
    # use the legacy interface for Python 2.3 support
    encoded_bytes = base64.encodestring(auth_bytes)
    # convert back to a string
    encoded = encoded_bytes.decode()
    # strip the trailing carriage return
    return encoded.replace('\n', '') 

def apply_config(self):
        """
        Apply global configuration object based on the configuration values.

        At this point this only applies to the JSON library which is used.
        """
        if not self.__config:
            # parse() hasn't been called yet. We should probably throw here
            return

        # Set json library based on the config value. If "auto" is provided this means we use
        # default behavior which is try to use ujson and if that's not available fall back to
        # stdlib json
        json_library = self.json_library
        current_json_library = scalyr_util.get_json_lib()

        if json_library != "auto" and json_library != current_json_library:
            self.__logger.debug(
                'Changing JSON library from "%s" to "%s"'
                % (current_json_library, json_library)
            )
            scalyr_util.set_json_lib(json_library) 

def __init__(
        self,
        k8s_log_configs,
        logger,
        rename_no_original,
        rename_logfile=None,
        parse_format="json",
    ):
        """
        @param k8s_log_configs: The config snippets from the configuration
        @param logger: A scalyr logger
        @param rename_no_original: A bool, used to prevent the original log file name from being added to the attributes.
        @param rename_logfile: A value for renaming a logfile - can contain variable substitutions
        @param parse_format: The parse format of this log config
        """

        if rename_logfile is None:
            rename_logfile = "/${container_runtime}/${container_name}.log"

        self.__k8s_log_configs = k8s_log_configs
        self._logger = logger
        self.__rename_logfile = rename_logfile
        self.__rename_no_original = rename_no_original
        self.__parse_format = parse_format 

def _to_python(self, value, state):
        try:
            return parse(value)
        except (ValueError, TypeError):
            if self.if_invalid != formencode.api.NoDefault:
                return self.if_invalid
            else:
                raise 

def _download_url(self, scheme, url, tmpdir):
        # Determine download filename
        #
        name, fragment = egg_info_for_url(url)
        if name:
            while '..' in name:
                name = name.replace('..', '.').replace('\\', '_')
        else:
            name = "__downloaded__"  # default if URL has no path contents

        if name.endswith('.egg.zip'):
            name = name[:-4]  # strip the extra .zip before download

        filename = os.path.join(tmpdir, name)

        # Download the file
        #
        if scheme == 'svn' or scheme.startswith('svn+'):
            return self._download_svn(url, filename)
        elif scheme == 'git' or scheme.startswith('git+'):
            return self._download_git(url, filename)
        elif scheme.startswith('hg+'):
            return self._download_hg(url, filename)
        elif scheme == 'file':
            return urllib.request.url2pathname(urllib.parse.urlparse(url)[2])
        else:
            self.url_ok(url, True)  # raises error if not allowed
            return self._attempt_download(url, filename) 

def local_open(url):
    """Read a local path, with special support for directories"""
    scheme, server, path, param, query, frag = urllib.parse.urlparse(url)
    filename = urllib.request.url2pathname(path)
    if os.path.isfile(filename):
        return urllib.request.urlopen(url)
    elif path.endswith('/') and os.path.isdir(filename):
        files = []
        for f in os.listdir(filename):
            filepath = os.path.join(filename, f)
            if f == 'index.html':
                with open(filepath, 'r') as fp:
                    body = fp.read()
                break
            elif os.path.isdir(filepath):
                f += '/'
            files.append('<a href="{name}">{name}</a>'.format(name=f))
        else:
            tmpl = ("<html><head><title>{url}</title>"
                "</head><body>{files}</body></html>")
            body = tmpl.format(url=url, files='\n'.join(files))
        status, message = 200, "OK"
    else:
        status, message, body = 404, "Path not found", "Not found"

    headers = {'content-type': 'text/html'}
    body_stream = six.StringIO(body)
    return urllib.error.HTTPError(url, status, message, headers, body_stream) 

def absolute_url(self, url):
        """Return the absolute URL made from the current URL and ``url``.
        The current URL is only used to provide any missing components of
        ``url``, as in the `.urljoin() method of urllib.parse
        <https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin>`__.
        """
        return urllib.parse.urljoin(self.get_url(), url) 

def urlquote(url, safe=b"/"):
    try:
        return six.moves.urllib.parse.quote(str(url), safe=safe)
    except UnicodeEncodeError:
        return six.moves.urllib.parse.quote(url.encode('utf-8'), safe=safe) 

def urlquoteplus(url, safe=b""):
    try:
        return six.moves.urllib.parse.quote_plus(str(url), safe=safe)
    except UnicodeEncodeError:
        return six.moves.urllib.parse.quote_plus(url.encode('utf-8'), safe=safe) 

def ago_string(s):
    try:
        return ago(parse(s, ignoretz=True))
    except (ValueError, AttributeError, TypeError):
        return 'unknown' 

def parse_requirement_arg(spec):
    try:
        return Requirement.parse(spec)
    except ValueError:
        raise DistutilsError(
            "Not a URL, existing file, or requirement spec: %r" % (spec,)
        ) 

def __build_request_data(data, request_format):
        if not data:
            return None

        if request_format == 'json':
            request_string = 'request=' + json.dumps(data)
        elif request_format == 'python':
            request_string = 'request=' + str(data)

        request_string = urllib.parse.quote(request_string, safe="{[]}\"=, :")

        return request_string.encode() 

def __build_request_path(self, query_params=None):
        path = self.web_api_base + '?'

        if not query_params:
            query_params = {}

        query_params.update({
            '_username': self.username,
            '_secret': self.secret
        })

        query_string = urllib.parse.urlencode(query_params)

        path += query_string
        return path 

def parse_url(url):
    try:
        from urlparse import urlparse
    except ImportError:
        # pylint: disable=import-error,no-name-in-module
        from urllib.parse import urlparse
    return urlparse(url) 

def find_python_interpreter(version_spec):
    import pkg_resources

    try:
        # Requirement.parse wants a package name, so we use 'python'
        # here, but anything would do.
        req = pkg_resources.Requirement.parse("python%s" % version_spec)
    except pkg_resources.RequirementParseError:
        raise ValueError(version_spec)
    python_interps = {ver: path for path, ver in python_interpreters()}
    matching = list(req.specifier.filter(sorted(python_interps)))
    if matching:
        matching_ver = matching[0]
        return python_interps[matching_ver], matching_ver
    return None 

def system():
    authorize()

    task_list = scheduler.get_task_list()

    throttled_providers = list_throttled_providers()
    
    try:
        with io.open(os.path.join(args.config_dir, 'config', 'releases.txt'), 'r', encoding='UTF-8') as f:
            releases = ast.literal_eval(f.read())
    except Exception as e:
        releases = []
        logging.exception(
            'BAZARR cannot parse releases caching file: ' + os.path.join(args.config_dir, 'config', 'releases.txt'))
    
    sonarr_version = get_sonarr_version()
    
    radarr_version = get_radarr_version()
    
    page_size = int(settings.general.page_size)
    
    return template('system', bazarr_version=bazarr_version,
                    sonarr_version=sonarr_version, radarr_version=radarr_version,
                    operating_system=platform.platform(), python_version=platform.python_version(),
                    config_dir=args.config_dir, bazarr_dir=os.path.normcase(os.path.dirname(os.path.dirname(__file__))),
                    base_url=base_url, task_list=task_list, page_size=page_size, releases=releases,
                    current_port=settings.general.port, throttled_providers=throttled_providers) 

def test_url(protocol, url):
    authorize()
    url = protocol + "://" + six.moves.urllib.parse.unquote(url)
    try:
        result = requests.get(url, allow_redirects=False, verify=False).json()['version']
    except Exception as e:
        logging.exception('BAZARR cannot successfully contact this URL: ' + url)
        return dict(status=False)
    else:
        return dict(status=True, version=result) 

def test_notification(protocol, provider):
    authorize()
    provider = six.moves.urllib.parse.unquote(provider)
    apobj = apprise.Apprise()
    apobj.add(protocol + "://" + provider)
    
    apobj.notify(
        title='Bazarr test notification',
        body=('Test notification')
    ) 

def from_url(cls, url):
        "Construct a (possibly null) ContentChecker from a URL"
        fragment = urllib.parse.urlparse(url)[-1]
        if not fragment:
            return ContentChecker()
        match = cls.pattern.search(fragment)
        if not match:
            return ContentChecker()
        return cls(**match.groupdict()) 

def egg_info_for_url(url):
    parts = urllib.parse.urlparse(url)
    scheme, server, path, parameters, query, fragment = parts
    base = urllib.parse.unquote(path.split('/')[-1])
    if server == 'sourceforge.net' and base == 'download':  # XXX Yuck
        base = urllib.parse.unquote(path.split('/')[-2])
    if '#' in base:
        base, fragment = base.split('#', 1)
    return base, fragment 

def parse_requirement_arg(spec):
    try:
        return Requirement.parse(spec)
    except ValueError:
        raise DistutilsError(
            "Not a URL, existing file, or requirement spec: %r" % (spec,)
        ) 

def absolute_url(self, url):
        """Return the absolute URL made from the current URL and ``url``.
        The current URL is only used to provide any missing components of
        ``url``, as in the `.urljoin() method of urllib.parse
        <https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin>`__.
        """
        return urllib.parse.urljoin(self.url, url)