Python urllib.error.HTTPError() Examples

def login(server, username, password, alias) -> None:
    data = {
        "username": username,
        "password": password,
    }
    try:
        with urlopen(f"{server}/_matrix/maubot/v1/auth/login",
                     data=json.dumps(data).encode("utf-8")) as resp_data:
            resp = json.load(resp_data)
            config["servers"][server] = resp["token"]
            if not config["default_server"]:
                print(Fore.CYAN, "Setting", server, "as the default server")
                config["default_server"] = server
            if alias:
                config["aliases"][alias] = server
            save_config()
            print(Fore.GREEN + "Logged in successfully")
    except HTTPError as e:
        try:
            err = json.load(e)
        except json.JSONDecodeError:
            err = {}
        print(Fore.RED + err.get("error", str(e)) + Fore.RESET) 

def _retried_request(self, url, func, data=None):
        retry_sleep_seconds = 1
        while True:
            try:
                if data is not None:
                    return func(url, data=data)
                return func(url)
            except HTTPError as e:
                if 500 <= e.code <= 599:
                    print('Error {}, retrying {} in {}s'.format(e.code, url, retry_sleep_seconds))
                elif e.code == 400 and e.reason == 'service in progress':
                    print('Service in progress, retrying {} in {}s'.format(url, retry_sleep_seconds))
                else:
                    raise e
                time.sleep(retry_sleep_seconds)
                # increase sleep time up to one minute to avoid hammering
                # the server in case of real problems
                if (retry_sleep_seconds % 60):
                    retry_sleep_seconds += 1 

def get_dstrepos(self, project):
        url = osc.core.makeurl(self.apiurl, ('source', project, '_meta'))
        try:
            root = ET.parse(osc.core.http_GET(url)).getroot()
        except HTTPError:
            return None

        repos = set()
        for repo in root.findall('repository'):
            name = repo.attrib['name']
            if project in REPO_WHITELIST and name not in REPO_WHITELIST[project]:
                continue

            for node in repo.findall('arch'):
                arch = node.text

                if project in ARCH_WHITELIST and arch not in ARCH_WHITELIST[project]:
                    continue

                if project in ARCH_BLACKLIST and arch in ARCH_BLACKLIST[project]:
                    continue

                repos.add((name, arch))

        return repos 

def _retried_request(self, url, func, data=None):
        retry_sleep_seconds = 1
        while True:
            try:
                if data is not None:
                    return func(url, data=data)
                return func(url)
            except HTTPError as e:
                if 500 <= e.code <= 599:
                    print('Error {}, retrying {} in {}s'.format(e.code, url, retry_sleep_seconds))
                elif e.code == 400 and e.reason == 'service in progress':
                    print('Service in progress, retrying {} in {}s'.format(url, retry_sleep_seconds))
                else:
                    raise e
                time.sleep(retry_sleep_seconds)
                # increase sleep time up to one minute to avoid hammering
                # the server in case of real problems
                if (retry_sleep_seconds % 60):
                    retry_sleep_seconds += 1 

def has_whitelist_warnings( self, source_project, source_package, target_project, target_package ):
        # this checks if this is a submit to an product project and it has warnings for non-whitelisted permissions/files
        found_entries = set()
        url = osc.core.makeurl(self.apiurl, ['build', target_project])
        xml = ET.parse(osc.core.http_GET(url)).getroot()
        for f in xml.findall('entry'):
            # we check all repos in the source project for errors that exist in the target project
            repo = f.attrib['name']
            query = { 'last': 1, }
            for arch in target_archs(self.apiurl, source_project, repo):
                url = osc.core.makeurl(self.apiurl, ['build', source_project, repo, arch, source_package, '_log'], query = query)
                try:
                    result = osc.core.http_GET(url)
                    contents = str(result.read())
                    for entry in self.bad_rpmlint_entries:
                        if (': W: ' + entry in contents) and not (entry in found_entries):
                            self.logger.info(f'found missing whitelist for warning: {entry}')
                            found_entries.add(entry)
                except HTTPError as e:
                    self.logger.info('ERROR in URL %s [%s]' % (url, e))
        return found_entries 

def add_explicit_disable(self, wipebinaries=False):

        self._init_biarch_packages()

        resulturl = self.makeurl(['source', self.project])
        result = ET.fromstring(self.cached_GET(resulturl))

        for pkg in self.packages:

            changed = False

            logger.debug("processing %s", pkg)
            if not pkg in self.package_metas:
                logger.error("%s not found", pkg)
                continue
            pkgmeta = self.package_metas[pkg]

            build = pkgmeta.findall("./build")
            if not build:
                logger.debug('disable %s for %s', pkg, self.arch)
                bn = pkgmeta.find('build')
                if bn is None:
                    bn = ET.SubElement(pkgmeta, 'build')
                ET.SubElement(bn, 'disable', { 'arch': self.arch })
                changed = True

            if changed:
                try:
                    pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta'])
                    self.http_PUT(pkgmetaurl, data=ET.tostring(pkgmeta))
                    if self.caching:
                        self._invalidate__cached_GET(pkgmetaurl)
                    if wipebinaries:
                        self.http_POST(self.makeurl(['build', self.project], {
                            'cmd': 'wipe',
                            'arch': self.arch,
                            'package': pkg }))
                except HTTPError as e:
                    logger.error('failed to update %s: %s', pkg, e) 

def binaries_of_product(self, project, product, repo=None, arch=None):
        if repo is None:
            repo = self.project.product_repo
        if arch is None:
            arch = self.project.product_arch

        url = self.api.makeurl(['build', project, repo, arch, product])
        try:
            f = self.api.retried_GET(url)
        except HTTPError:
            return []

        ret = []
        root = ET.parse(f).getroot()
        for binary in root.findall('binary'):
            ret.append(binary.get('filename'))

        return ret 

def find_cf_issues(domains):

    error_domains = []

    for domain in domains:
        try:
            response = urlopen('http://' + domain)
        except HTTPError as e:
            if e.code == 403 and 'Bad request' in e.fp.read():
                try:
                    response = urlopen('https://' + domain)
                except URLError as e:
                    if 'handshake' in str(e).lower() or e.code == 403 and 'Bad request' in e.fp.read():
                        error_domains.append(domain)
                except:
                    pass
        except:
            pass

    return error_domains

# add a domain to CloudFront 

def _make_request(self, opener, request, timeout=None):
        """Make the API call and return the response. This is separated into
           it's own function, so we can mock it easily for testing.

        :param opener:
        :type opener:
        :param request: url payload to request
        :type request: urllib.Request object
        :param timeout: timeout value or None
        :type timeout: float
        :return: urllib response
        """
        timeout = timeout or self.timeout
        try:
            return opener.open(request, timeout=timeout)
        except HTTPError as err:
            exc = handle_error(err)
            return exc 

def _setprio(self, status, priority):
        """
        Set prios for requests that are still in review
        :param project: project to check

        """
        message = 'raising priority for %s' % status.get('name')
        for r in status.findall('missing_reviews/review'):
            reqid = r.get('request')
            req = osc.core.get_request(self.api.apiurl, reqid)
            if req.priority == priority:
                continue
            query = { 'cmd': 'setpriority', 'priority': priority }
            url = osc.core.makeurl(self.api.apiurl, ['request', reqid], query)
            print(f"raising priority of {r.get('package')} [{r.get('request')}] to {priority}")
            try:
                osc.core.http_POST(url, data=message)
            except HTTPError as e:
                print(e) 

def _filter_packages_by_time(self, packages):
        x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, '000product', '_history'], {'limit': '1'})))
        producttime = int(x.find('./revision/time').text)
        for pkg in packages:
            try:
                x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, pkg, '_history'], {'rev': '1'})))
            # catch deleted packages
            except HTTPError as e:
                if e.code == 404:
                    continue
                raise e

            packagetime = int(x.find('./revision/time').text)
#            if producttime > packagetime:
#                continue
            yield pkg 

def HTML(self, check):
        try:
            if self.UserAgent != None:
                page_html = urlopen(Request(
                        self.target_url,
                        headers={"User-Agent":self.UserAgent}),
                        timeout=self.TimeOut).read().decode("utf-8")

            #If not, the default will be used
            else:
                page_html = urlopen(
                        self.target_url,
                        timeout=self.TimeOut).read().decode("utf-8")

        except HTTPError:
            page_html = "Can't get page source code"

        if self.verbose == True:
            print(" [+] Source code got from %s" % self.target_url)
            print("----START" + "-"*71)
            print(page_html)
            print("----END" + "-"*73)

        return(page_html) 

def _http_request(url, headers=None, time_out=10):
    """Perform an HTTP request and return request"""
    log(0, 'Request URL: {url}', url=url)

    try:
        if headers:
            request = Request(url, headers=headers)
        else:
            request = Request(url)
        req = urlopen(request, timeout=time_out)
        log(0, 'Response code: {code}', code=req.getcode())
        if 400 <= req.getcode() < 600:
            raise HTTPError('HTTP %s Error for url: %s' % (req.getcode(), url), response=req)
    except (HTTPError, URLError) as err:
        log(2, 'Download failed with error {}'.format(err))
        if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))):  # Internet down, try again?
            return _http_request(url, headers, time_out)
        return None

    return req 

def check_image_bdeps(self, project, arch):
        for dvd in ('000product:openSUSE-dvd5-dvd-{}'.format(arch), 'Test-DVD-{}'.format(arch)):
            try:
                url = makeurl(self.api.apiurl, ['build', project, 'images', arch, dvd, '_buildinfo'])
                root = ET.parse(http_GET(url)).getroot()
            except HTTPError as e:
                if e.code == 404:
                    continue
                raise
            for bdep in root.findall('bdep'):
                if 'name' not in bdep.attrib:
                    continue
                b = bdep.attrib['name']
                if b not in self.bin2src:
                    print("{} not found in bin2src".format(b))
                    continue
                b = self.bin2src[b]
                self.pkgdeps[b] = 'MYdvd{}'.format(self.api.rings.index(project))
            break 

def load_request(self, request_id):
        if not _is_int(request_id):
            return None

        url = makeurl(self.api.apiurl, ['request', str(request_id)])
        try:
            f = http_GET(url)
        except HTTPError:
            return None

        root = ET.parse(f).getroot()

        if root.get('id', None) != str(request_id):
            return None

        return root 

def package_source_hash(apiurl, project, package, revision=None):
    query = {}
    if revision:
        query['rev'] = revision

    # Will not catch packages that previous had a link, but no longer do.
    if package_source_link_copy(apiurl, project, package):
        query['expand'] = 1

    try:
        url = makeurl(apiurl, ['source', project, package], query)
        root = ETL.parse(http_GET(url)).getroot()
    except HTTPError as e:
        if e.code == 400 or e.code == 404:
            # 400: revision not found, 404: package not found.
            return None

        raise e

    if revision and root.find('error') is not None:
        # OBS returns XML error instead of HTTP 404 if revision not found.
        return None

    from osclib.util import sha1_short
    return sha1_short(root.xpath('entry[@name!="_link"]/@md5')) 

def package_source_hash(apiurl, project, package, revision=None):
    query = {}
    if revision:
        query['rev'] = revision

    # Will not catch packages that previous had a link, but no longer do.
    if package_source_link_copy(apiurl, project, package):
        query['expand'] = 1

    try:
        url = makeurl(apiurl, ['source', project, package], query)
        root = ETL.parse(http_GET(url)).getroot()
    except HTTPError as e:
        if e.code == 400 or e.code == 404:
            # 400: revision not found, 404: package not found.
            return None

        raise e

    if revision and root.find('error') is not None:
        # OBS returns XML error instead of HTTP 404 if revision not found.
        return None

    from osclib.util import sha1_short
    return sha1_short(root.xpath('entry[@name!="_link"]/@md5')) 

def can_accept_review(self, request_id):
        """return True if there is a new review for the specified reviewer"""
        states = set()
        url = osc.core.makeurl(self.apiurl, ('request', str(request_id)))
        try:
            root = ET.parse(osc.core.http_GET(url)).getroot()
            if self.review_user and self._has_open_review_by(root, 'by_user', self.review_user):
                return True
            if self.review_group and self._has_open_review_by(root, 'by_group', self.review_group):
                return True
        except HTTPError as e:
            print('ERROR in URL %s [%s]' % (url, e))
        return False 

def _package_source_parse(self, project, package, revision=None, repository=None):
        query = {'view': 'info', 'parse': 1}
        if revision:
            query['rev'] = revision
        if repository:
            query['repository'] = repository
        url = osc.core.makeurl(self.apiurl, ['source', project, package], query)

        ret = {'name': None, 'version': None}

        try:
            xml = ET.parse(osc.core.http_GET(url)).getroot()
        except HTTPError as e:
            self.logger.error('ERROR in URL %s [%s]' % (url, e))
            return ret

        if xml.find('error') is not None:
            self.logger.error("%s/%s/%s: %s", project, package, repository, xml.find('error').text)
            return ret

        # ET boolean check fails.
        if xml.find('name') is not None:
            ret['name'] = xml.find('name').text

        if xml.find('version') is not None:
            ret['version'] = xml.find('version').text

        if xml.find('filename') is not None:
            ret['filename'] = xml.find('filename').text

        self.logger.debug("%s/%s/%s: %s", project, package, repository, ret)

        return ret 

def source_file_load(apiurl, project, package, filename, revision=None):
    query = {'expand': 1}
    if revision:
        query['rev'] = revision
    url = makeurl(apiurl, ['source', project, package, filename], query)
    try:
        return decode_it(http_GET(url).read())
    except HTTPError:
        return None 

def repository_arch_state(apiurl, project, repository, arch):
    # just checking the mtimes of the repository's binaries
    url = makeurl(apiurl, ['build', project, repository, arch, '_repository'])
    from osclib.util import sha1_short
    try:
        return sha1_short(http_GET(url).read())
    except HTTPError as e:
        # e.g. staging projects inherit the project config from 'ports' repository.
        # but that repository does not contain the archs we want, as such it has no state
        if e.code != 404:
            raise e 

def _last_build_success(self, src_project, tgt_project, src_package, rev):
        """Return the last build success XML document from OBS."""
        try:
            query = { 'lastsuccess' : 1,
                    'package' : src_package,
                    'pathproject' : tgt_project,
                    'srcmd5' : rev }
            url = osc.core.makeurl(self.apiurl, ('build', src_project, '_result'), query)
            return ET.parse(osc.core.http_GET(url)).getroot()
        except HTTPError as e:
            if e.code != 404:
                self.logger.error('ERROR in URL %s [%s]' % (url, e))
                raise
            pass
        return None 

def _getmtimes(self, prj, pkg, repo, arch):
        """ returns a dict of filename: mtime """
        url = osc.core.makeurl(self.apiurl, ('build', prj, repo, arch, pkg))
        try:
            root = ET.parse(osc.core.http_GET(url)).getroot()
        except HTTPError:
            return None

        return dict([(node.attrib['filename'], node.attrib['mtime']) for node in root.findall('binary')])

    # modified from repochecker 

def _check_matching_srcmd5(self, project, package, rev, history_limit = 5):
        """check if factory sources contain the package and revision. check head and history"""
        self.logger.debug("checking %s in %s" % (package, project))
        try:
            osc.core.show_package_meta(self.apiurl, project, package)
        except (HTTPError, URLError):
            self.logger.debug("new package")
            return None

        si = self.get_sourceinfo(project, package)
        if rev == si.verifymd5:
            self.logger.debug("srcmd5 matches")
            return True

        if history_limit:
            self.logger.debug("%s not the latest version, checking history", rev)
            u = osc.core.makeurl(self.apiurl, [ 'source', project, package, '_history' ], { 'limit': history_limit })
            try:
                r = osc.core.http_GET(u)
            except HTTPError as e:
                self.logger.debug("package has no history!?")
                return None

            root = ET.parse(r).getroot()
            # we need this complicated construct as obs doesn't honor
            # the 'limit' parameter use above for obs interconnect:
            # https://github.com/openSUSE/open-build-service/issues/2545
            for revision, i in zip(reversed(root.findall('revision')), count()):
                node = revision.find('srcmd5')
                if node is None:
                    continue
                self.logger.debug("checking %s" % node.text)
                if node.text == rev:
                    self.logger.debug("got it, rev %s" % revision.get('rev'))
                    return True
                if i == history_limit:
                    break

            self.logger.debug("srcmd5 not found in history either")

        return False 

def package_version(apiurl, project, package):
    try:
        url = makeurl(apiurl, ['source', project, package, '_history'], {'limit': 1})
        root = ETL.parse(http_GET(url)).getroot()
    except HTTPError as e:
        if e.code == 404:
            return False

        raise e

    return str(root.xpath('(//version)[last()]/text()')[0]) 

def _get_linktarget(self, src_project, src_package):

        query = {}
        url = osc.core.makeurl(self.apiurl, ('source', src_project, src_package), query=query)
        try:
            root = ET.parse(osc.core.http_GET(url)).getroot()
        except HTTPError:
            return (None, None)

        if root is not None:
            linkinfo = root.find("linkinfo")
            if linkinfo is not None:
                return (linkinfo.get('project'), linkinfo.get('package'))

        return (None, None) 

def _get_sourceinfo(apiurl, project, package, rev=None):
        query = { 'view': 'info' }
        if rev is not None:
            query['rev'] = rev
        url = osc.core.makeurl(apiurl, ('source', project, package), query=query)
        try:
            return ET.parse(osc.core.http_GET(url)).getroot()
        except (HTTPError, URLError):
            return None 

def has_staging(self, project):
        try:
            url = osc.core.makeurl(self.apiurl, ('staging', project, 'staging_projects'))
            osc.core.http_GET(url)
            return True
        except HTTPError as e:
            if e.code != 404:
                self.logger.error('ERROR in URL %s [%s]' % (url, e))
                raise
            pass
        return False 

def _load_lookup_file(self, prj):
        try:
            return osc.core.http_GET(osc.core.makeurl(self.apiurl,
                                ['source', prj, '00Meta', 'lookup.yml']))
        except HTTPError as e:
            # in case the project doesn't exist yet (like sle update)
            if e.code != 404:
                raise e
            return None 

def entity_source_link(apiurl, project, package=None):
    try:
        if package:
            parts = ['source', project, package, '_link']
        else:
            parts = ['source', project, '_meta']
        url = makeurl(apiurl, parts)
        root = ETL.parse(http_GET(url)).getroot()
    except HTTPError as e:
        if e.code == 404:
            return None

        raise e

    return root if package else root.find('link')