Python urllib2.HTTPError() Examples

def jenkins(url, port):
    try:
        cli_port = False
        ctx = ssl.create_default_context()
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
        try:
            output = urllib2.urlopen('https://'+url+':'+port+"/jenkins/", context=ctx, timeout=8).info()
            cli_port = int(output['X-Jenkins-CLI-Port'])
        except urllib2.HTTPError, e:
            if e.getcode() == 404:
                try:
                    output = urllib2.urlopen('https://'+url+':'+port, context=ctx, timeout=8).info()
                    cli_port = int(output['X-Jenkins-CLI-Port'])
                except:
                    pass
        except:
            pass 

def post_request(vals, url):
    """
    Build a post request.

    Args:
        vals: Dictionary of (field, values) for the POST
            request.
        url: URL to send the data to.

    Returns:
        Dictionary of JSON response or error info.
    """
    # Build the request and send to server
    data = urllib.urlencode(vals)
    
    try:
        request  = urllib2.Request(url, data)
        response = urllib2.urlopen(request)
    except urllib2.HTTPError, err:
        return {"error": err.reason, "error_code": err.code} 

def _make_request(self, opener, request, timeout=None):
        """Make the API call and return the response. This is separated into
           it's own function, so we can mock it easily for testing.

        :param opener:
        :type opener:
        :param request: url payload to request
        :type request: urllib.Request object
        :param timeout: timeout value or None
        :type timeout: float
        :return: urllib response
        """
        timeout = timeout or self.timeout
        try:
            return opener.open(request, timeout=timeout)
        except HTTPError as err:
            exc = handle_error(err)
            return exc 

def query(owner, name):
    if fake:
        print("    {0}/{1}: ok".format(owner, name))
        return (random.randint(1, 1000), random.randint(1, 300))
    else:
        try:
            req = urllib2.Request(
                "https://api.github.com/repos/{0}/{1}".format(owner, name)
            )
            if user is not None and token is not None:
                b64 = base64.encodestring("{0}:{1}".format(user, token)).replace(
                    "\n", ""
                )
                req.add_header("Authorization", "Basic {0}".format(b64))
            u = urllib2.urlopen(req)
            j = json.load(u)
            t = datetime.datetime.strptime(j["updated_at"], "%Y-%m-%dT%H:%M:%SZ")
            days = max(int((now - t).days), 0)
            print("    {0}/{1}: ok".format(owner, name))
            return (int(j["stargazers_count"]), days)
        except urllib2.HTTPError as e:
            print("    {0}/{1}: FAILED".format(owner, name))
            return (None, None) 

def check_multiple_specfiles(self, project, package):
        try:
            url = makeurl(self.apiurl, ['source', project, package], { 'expand': '1' } )
        except HTTPError as e:
            if e.code == 404:
                return None
            raise e
        root = ET.fromstring(http_GET(url).read())
        data = {}
        linkinfo = root.find('linkinfo')
        if linkinfo is not None:
            data['linkinfo'] = linkinfo.attrib['package']
        else:
            data['linkinfo'] = None

        files = [ entry.get('name').replace('.spec', '') for entry in root.findall('entry') if entry.get('name').endswith('.spec') ]
        data['specs'] = files

        if len(files) > 1:
            return data
        else:
            return False 

def add_explicit_disable(self, wipebinaries=False):

        self._init_biarch_packages()

        resulturl = self.makeurl(['source', self.project])
        result = ET.fromstring(self.cached_GET(resulturl))

        for pkg in self.packages:

            changed = False

            logger.debug("processing %s", pkg)
            if not pkg in self.package_metas:
                logger.error("%s not found", pkg)
                continue
            pkgmeta = self.package_metas[pkg]

            build = pkgmeta.findall("./build")
            if not build:
                logger.debug('disable %s for %s', pkg, self.arch)
                bn = pkgmeta.find('build')
                if bn is None:
                    bn = ET.SubElement(pkgmeta, 'build')
                ET.SubElement(bn, 'disable', { 'arch': self.arch })
                changed = True

            if changed:
                try:
                    pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta'])
                    self.http_PUT(pkgmetaurl, data=ET.tostring(pkgmeta))
                    if self.caching:
                        self._invalidate__cached_GET(pkgmetaurl)
                    if wipebinaries:
                        self.http_POST(self.makeurl(['build', self.project], {
                            'cmd': 'wipe',
                            'arch': self.arch,
                            'package': pkg }))
                except HTTPError as e:
                    logger.error('failed to update %s: %s', pkg, e) 

def lookup(self, name, host_override=None):
        """
        Looks up a name from the DNSChain server. Throws exception if the
        data is not valid JSON or if the namecoin entry does not exist in the
        blockchain.

        @param name: The name to lookup, e.g. 'id/dionyziz', note this $NAMESPACE/$NAME
        format is not guaranteed.  Additionally the caller must perform appropriate url
        encoding _before_ the name is passed to urllib2.urlopen
        """
        if host_override is not None:
            self.headers['Host'] = host_override
        full_url = "http://%s/%s" % (self.addr, name)
        request = urllib2.Request(full_url, None, self.headers)
        try:
            response = urllib2.urlopen(request)
        except urllib2.HTTPError, e:
            if e.code == 404:
                e = DataNotFound(e, name, self.headers['Host'])
            if e.code < 200 or e.code > 299:
                self._log.debug("%s" % (e.msg,), exc_info=True)
                raise e 

def _filter_packages_by_time(self, packages):
        x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, '000product', '_history'], {'limit': '1'})))
        producttime = int(x.find('./revision/time').text)
        for pkg in packages:
            try:
                x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, pkg, '_history'], {'rev': '1'})))
            # catch deleted packages
            except HTTPError as e:
                if e.code == 404:
                    continue
                raise e

            packagetime = int(x.find('./revision/time').text)
#            if producttime > packagetime:
#                continue
            yield pkg 

def __init__(self, http_error):
    """Create a ServerRequestException from a given urllib2.HTTPError.

    Args:
      http_error: The HTTPError that the ServerRequestException will be
        based on.
    """
    error_details = None
    if http_error.fp:
      try:
        error_body = json.load(http_error.fp)
        error_details = ['%s: %s' % (detail['message'], detail['debug_info'])
                         for detail in error_body['error']['errors']]
      except (ValueError, TypeError, KeyError):
        pass
    if error_details:
      error_message = ('HTTP %s (%s) error when communicating with URL: %s.  '
                       'Details: %s' % (http_error.code, http_error.reason,
                                        http_error.filename, error_details))
    else:
      error_message = ('HTTP %s (%s) error when communicating with URL: %s.' %
                       (http_error.code, http_error.reason,
                        http_error.filename))
    super(ServerRequestException, self).__init__(error_message) 

def check_image_bdeps(self, project, arch):
        for dvd in ('000product:openSUSE-dvd5-dvd-{}'.format(arch), 'Test-DVD-{}'.format(arch)):
            try:
                url = makeurl(self.api.apiurl, ['build', project, 'images', arch, dvd, '_buildinfo'])
                root = ET.parse(http_GET(url)).getroot()
            except HTTPError as e:
                if e.code == 404:
                    continue
                raise
            for bdep in root.findall('bdep'):
                if 'name' not in bdep.attrib:
                    continue
                b = bdep.attrib['name']
                if b not in self.bin2src:
                    print("{} not found in bin2src".format(b))
                    continue
                b = self.bin2src[b]
                self.pkgdeps[b] = 'MYdvd{}'.format(self.api.rings.index(project))
            break 

def http_download(url, dest):
    """Safely download url to dest.

    Print error and exit if download fails.
    """
    try:
        print("GETting", url, "to", dest, "...", end=' ')
        f = urlopen(url)
        # Open our local file for writing
        with open(dest, "wb") as local_file:
            local_file.write(f.read())

    except HTTPError as e:
        print()
        print("HTTP Error:", e.code, url)
        sys.exit(1)

    except URLError as e:
        print()
        print("URL Error:", e.reason, url)
        sys.exit(1)
    print("done.") 

def freeze(self):
        """Main method"""
        flink = ET.Element('frozenlinks')

        fl = ET.SubElement(flink, 'frozenlink', {'project': self.factory})
        ignored_sources = self.receive_sources(fl)
        if self.debug:
            logging.debug("Dump ignored source")
            for source in ignored_sources:
                logging.debug("Ignored source: %s" % source)

        url = makeurl(self.apiurl, ['source', FCC, '_project', '_frozenlinks'], {'meta': '1'})
        l = ET.tostring(flink)
        try:
            http_PUT(url, data=l)
        except HTTPError as e:
            raise e 

def DoUpload(self):
    """Uploads the pagespeed entries."""

    pagespeed_yaml = ''
    if self.pagespeed:
      StatusUpdate('Uploading PageSpeed configuration.')
      pagespeed_yaml = self.pagespeed.ToYAML()
    try:
      self.rpcserver.Send('/api/appversion/updatepagespeed',
                          app_id=self.config.application,
                          version=self.config.version,
                          payload=pagespeed_yaml)
    except urllib2.HTTPError, err:









      if err.code != 404 or self.pagespeed is not None:
        raise 

def _http_request(url, headers=None, time_out=10):
    """Perform an HTTP request and return request"""
    log(0, 'Request URL: {url}', url=url)

    try:
        if headers:
            request = Request(url, headers=headers)
        else:
            request = Request(url)
        req = urlopen(request, timeout=time_out)
        log(0, 'Response code: {code}', code=req.getcode())
        if 400 <= req.getcode() < 600:
            raise HTTPError('HTTP %s Error for url: %s' % (req.getcode(), url), response=req)
    except (HTTPError, URLError) as err:
        log(2, 'Download failed with error {}'.format(err))
        if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))):  # Internet down, try again?
            return _http_request(url, headers, time_out)
        return None

    return req 

def query(owner, name):
    if fake:
        print '    {0}/{1}: ok'.format(owner, name)
        return (random.randint(1, 1000), random.randint(1, 300))
    else:
        try:
            req = urllib2.Request('https://api.github.com/repos/{0}/{1}'.format(owner, name))
            if user is not None and token is not None:
                b64 = base64.encodestring('{0}:{1}'.format(user, token)).replace('\n', '')
                req.add_header("Authorization", "Basic {0}".format(b64))
            u = urllib2.urlopen(req)
            j = json.load(u)
            t = datetime.datetime.strptime(j['updated_at'], "%Y-%m-%dT%H:%M:%SZ")
            days = max(int((now - t).days), 0)
            print '    {0}/{1}: ok'.format(owner, name)
            return (int(j['stargazers_count']), days)
        except urllib2.HTTPError, e:
            print '    {0}/{1}: FAILED'.format(owner, name)
            return (None, None) 

def find_cf_issues(domains):

    error_domains = []

    for domain in domains:
        try:
            response = urlopen('http://' + domain)
        except HTTPError as e:
            if e.code == 403 and 'Bad request' in e.fp.read():
                try:
                    response = urlopen('https://' + domain)
                except URLError as e:
                    if 'handshake' in str(e).lower() or e.code == 403 and 'Bad request' in e.fp.read():
                        error_domains.append(domain)
                except:
                    pass
        except:
            pass

    return error_domains

# add a domain to CloudFront 

def reverseip(url):
    """return domains from given the same server"""

    # get only domain name
    url = urlparse(url).netloc if urlparse(url).netloc != '' else urlparse(url).path.split("/")[0]

    source = "http://domains.yougetsignal.com/domains.php"
    useragent = useragents.get()
    contenttype = "application/x-www-form-urlencoded; charset=UTF-8"

    # POST method
    opener = urllib2.build_opener(
        urllib2.HTTPHandler(), urllib2.HTTPSHandler())
    data = urllib.urlencode([('remoteAddress', url), ('key', '')])

    request = urllib2.Request(source, data)
    request.add_header("Content-type", contenttype)
    request.add_header("User-Agent", useragent)

    try:
        result = urllib2.urlopen(request).read()

    except urllib2.HTTPError, e:
        print >> sys.stderr, "[{}] HTTP error".format(e.code) 

def Run(self):
    """Executes the requested action.

    Catches any HTTPErrors raised by the action and prints them to stderr.

    Returns:
      1 on error, 0 if successful.
    """
    try:
      self.action(self)
    except urllib2.HTTPError, e:
      body = e.read()
      if self.wrap_server_error_message:
        error_format = ('Error %d: --- begin server output ---\n'
                        '%s\n--- end server output ---')
      else:
        error_format = 'Error %d: %s'

      print >>self.error_fh, (error_format % (e.code, body.rstrip('\n')))
      return 1 

def check_image_bdeps(self, project, arch):
        for dvd in ('000product:openSUSE-dvd5-dvd-{}'.format(arch), 'Test-DVD-{}'.format(arch)):
            try:
                url = makeurl(self.api.apiurl, ['build', project, 'images', arch, dvd, '_buildinfo'])
                root = ET.parse(http_GET(url)).getroot()
            except HTTPError as e:
                if e.code == 404:
                    continue
                raise
            for bdep in root.findall('bdep'):
                if 'name' not in bdep.attrib:
                    continue
                b = bdep.attrib['name']
                if b not in self.bin2src:
                    print("{} not found in bin2src".format(b))
                    continue
                b = self.bin2src[b]
                self.pkgdeps[b] = 'MYdvd{}'.format(self.api.rings.index(project))
            break 

def search(self, query, pages=10):
        """search and return an array of urls"""

        urls = []

        try:
            for url in google.search(query, start=0, stop=pages):
                urls.append(url)
        except HTTPError:
            exit("[503] Service Unreachable")
        except URLError:
            exit("[504] Gateway Timeout")
        except:
            exit("Unknown error occurred")
        else:
            return urls 

def _GetAuthCookie(self, auth_token):
    """Fetches authentication cookies for an authentication token.

    Args:
      auth_token: The authentication token returned by ClientLogin.

    Raises:
      HTTPError: If there was an error fetching the authentication cookies.
    """

    continue_location = "http://localhost/"
    args = {"continue": continue_location, "auth": auth_token}
    login_path = os.environ.get("APPCFG_LOGIN_PATH", "/_ah")
    req = self._CreateRequest("%s://%s%s/login?%s" %
                              (self.scheme, self.host, login_path,
                               urllib.urlencode(args)))
    try:
      response = self.opener.open(req)
    except urllib2.HTTPError, e:
      response = e 

def check_for_update():
  if os.path.exists(FILE_UPDATE):
    mtime = os.path.getmtime(FILE_UPDATE)
    last = datetime.utcfromtimestamp(mtime).strftime('%Y-%m-%d')
    today = datetime.utcnow().strftime('%Y-%m-%d')
    if last == today:
      return
  try:
    with open(FILE_UPDATE, 'a'):
      os.utime(FILE_UPDATE, None)
    request = urllib2.Request(
      CORE_VERSION_URL,
      urllib.urlencode({'version': __version__}),
    )
    response = urllib2.urlopen(request)
    with open(FILE_UPDATE, 'w') as update_json:
      update_json.write(response.read())
  except (urllib2.HTTPError, urllib2.URLError):
    pass 

def call_api(self, endpoint, method, headers=None, params=[], data=None, quiet=False):

        path = self.parse_path(endpoint, params)

        # If custom headers are not specified we can use the default headers
        if not headers:
            headers = self.headers
        # Send the request and receive the response
        if not self.quiet:
            print('\nSending ' + method + ' request to: ' + 'https://' +self.server_ip+self.base_uri+path+'\n')

        request = Request(
            'https://'+self.server_ip+self.base_uri+path, headers=headers)
        request.get_method = lambda: method
        try:
            #returns response object for opening url.
            return urlopen(request, data)
        except HTTPError as e:
            #an object which contains information similar to a request object
            return e

    # This method constructs the query string 

def is_source_innerlink(self, project, package):
        try:
            root = ET.fromstring(
                self.cached_GET(makeurl(self.apiurl,
                                 ['source', project, package, '_link']
                )))
            if root.get('project') is None and root.get('cicount'):
                return True
        except HTTPError as err:
            # if there is no link, it can't be a link
            if err.code == 404:
                return False
            raise 

def get_response(self, query):
        response = None
        try:
            # we can't use helpers.retrieve_url because of redirects
            # we need the cookie processor to handle redirects
            opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(CookieJar()))
            response = opener.open(query).read().decode('utf-8')
        except urllib_request.HTTPError as e:
            # if the page returns a magnet redirect, used in download_torrent
            if e.code == 302:
                response = e.url
        except Exception:
            pass
        return response 

def item_exists(self, project, package=None):
        """
        Return true if the given project or package exists
        """
        if package:
            url = makeurl(self.apiurl, ['source', project, package, '_meta'])
        else:
            url = makeurl(self.apiurl, ['source', project, '_meta'])
        try:
            http_GET(url)
        except HTTPError:
            return False
        return True 

def get_response(self, query):
        response = None
        try:
            # we can't use helpers.retrieve_url because of redirects
            # we need the cookie processor to handle redirects
            opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(CookieJar()))
            response = opener.open(query).read().decode('utf-8')
        except urllib_request.HTTPError as e:
            # if the page returns a magnet redirect, used in download_torrent
            if e.code == 302:
                response = e.url
        except Exception:
            pass
        return response 

def call_api(self, endpoint, method, headers=None, params=[], data=None, quiet=False):

        path = self.parse_path(endpoint, params)

        # If custom headers are not specified we can merge the default headers
        if not headers:
            headers = self.headers
        else:
            for key, value in self.headers.items():
                if headers.get( key,'') == '':
                    headers[ key ] = value

        # Send the request and receive the response
        if not self.quiet:
            print('\nSending ' + method + ' request to: ' + 'https://' +self.server_ip+self.base_uri+path+'\n')

        # This disables all SSL certificate verification
        context = ssl._create_unverified_context()
		
        request = Request(
            'https://'+self.server_ip+self.base_uri+path, headers=headers)
        request.get_method = lambda: method
        try:
            #returns response object for opening url.
            return urlopen(request, data, context=context)
        except HTTPError as e:
            #an object which contains information similar to a request object
            return e

    # This method constructs the query string 

def retried_GET(self, url):
        try:
            return http_GET(url)
        except HTTPError as e:
            if 500 <= e.code <= 599:
                print('Retrying {}'.format(url))
                time.sleep(1)
                return self.retried_GET(url)
            raise e 

def _read(self):
        url = makeurl(self.apiurl, ['source', self.lock, '_attribute', '%s:LockedBy' % self.ns])
        try:
            root = ET.parse(http_GET(url)).getroot()
        except HTTPError as e:
            if e.code == 404:
                return None
            raise e
        signature = None
        try:
            signature = root.find('.//value').text
        except (AttributeError, ValueError):
            pass
        return signature