Python urllib.request.HTTPError() Examples

def get_access_token(self, code, state=None):
        '''
        In callback url: http://host/callback?code=123&state=xyz
        use code and state to get an access token.
        '''
        kw = dict(client_id=self._client_id, client_secret=self._client_secret, code=code)
        if self._redirect_uri:
            kw['redirect_uri'] = self._redirect_uri
        if state:
            kw['state'] = state
        opener = build_opener(HTTPSHandler)
        request = Request('https://github.com/login/oauth/access_token', data=_encode_params(kw))
        request.get_method = _METHOD_MAP['POST']
        request.add_header('Accept', 'application/json')
        try:
            response = opener.open(request, timeout=TIMEOUT)
            r = _parse_json(response.read())
            if 'error' in r:
                raise ApiAuthError(str(r.error))
            return str(r.access_token)
        except HTTPError as e:
            raise ApiAuthError('HTTPError when get access token') 

def get_access_token(self, code, state=None):
        '''
        In callback url: http://host/callback?code=123&state=xyz

        use code and state to get an access token.        
        '''
        kw = dict(client_id=self._client_id, client_secret=self._client_secret, code=code)
        if self._redirect_uri:
            kw['redirect_uri'] = self._redirect_uri
        if state:
            kw['state'] = state
        opener = build_opener(HTTPSHandler)
        request = Request('https://github.com/login/oauth/access_token', data=_encode_params(kw))
        request.get_method = _METHOD_MAP['POST']
        request.add_header('Accept', 'application/json')
        try:
            response = opener.open(request, timeout=TIMEOUT)
            r = _parse_json(response.read())
            if 'error' in r:
                raise ApiAuthError(str(r.error))
            return str(r.access_token)
        except HTTPError as e:
            raise ApiAuthError('HTTPError when get access token') 

def download_fzf_binary(plat, arch, overwrite=False, access_token=None):
    bin_path = fzf_windows_bin_path if plat == 'windows' else fzf_bin_path
    if overwrite or not os.path.isfile(bin_path):
        asset = get_fzf_binary_url(plat, arch, access_token)
        url, ext = asset
        if access_token:
            url = '{0}?access_token={1}'.format(url, access_token)
        try:
            r = urllib2.urlopen(url)
        except urllib2.HTTPError as e:
            if e.code == 403 and e.info().get('X-RateLimit-Remaining') == 0:
                raise RuntimeError(
                    'GitHub rate limit reached. To increate the limit use '
                    '-g/--github-access-token option.\n  ' + str(e)
                )
            elif e.code == 401 and access_token:
                raise RuntimeError('Invalid GitHub access token.')
            raise
        extract(r, ext, bin_path)
        r.close()
    mode = os.stat(bin_path).st_mode
    if not (mode & 0o111):
        os.chmod(bin_path, mode | 0o111) 

def readSiteMap(self):
        pages = []
        try:
            # f = urlopen("http://www.codepool.biz/sitemap.xml")
            request = build_request("http://kb.dynamsoft.com/sitemap.xml")
            f = urlopen(request, timeout=3)
            xml = f.read()

            soup = BeautifulSoup(xml)
            urlTags = soup.find_all("url")

            print "The number of url tags in sitemap: ", str(len(urlTags))

            for sitemap in urlTags:
                link = sitemap.findNext("loc").text
                pages.append(link)

            f.close()
        except HTTPError, URLError:
            print URLError.code 

def crawlLinks(self, links, file=None):
        for link in links:
            if shutdown_event.isSet():
                return GAME_OVER

            status_code = 0

            try:
                request = build_request(link)
                f = urlopen(request)
                status_code = f.code
                f.close()
            except HTTPError, URLError:
                status_code = URLError.code

            if status_code == 404:
                if file != None:
                    file.write(link + '\n')

            print str(status_code), ':', link 

def submit_report(self, report_data):
        data = json.dumps(report_data).encode('utf8')
        self.register_cbt("Logger", "LOG_DEBUG", "Usage report data: {0}".format(data))
        url = None
        try:
            url = "http://" + self._cm_config["ServerAddress"] + ":" + \
                  str(self._cm_config["ServerPort"]) + "/api/submit"
            req = urllib2.Request(url=url, data=data)
            req.add_header("Content-Type", "application/json")
            res = urllib2.urlopen(req)
            if res.getcode() == 200:
                log = "Usage report successfully submitted to server {0}\n" \
                      "HTTP response code:{1}, msg:{2}" \
                    .format(url, res.getcode(), res.read())
                self.register_cbt("Logger", "LOG_INFO", log)
            else:
                self.register_cbt("Logger", "LOG_WARNING",
                                  "Usage report server indicated error "
                                  "code: {0}".format(res.getcode()))
        except (urllib2.HTTPError, urllib2.URLError) as error:
            log = "Usage report submission failed to server {0}. " \
                  "Error: {1}".format(url, error)
            self.register_cbt("Logger", "LOG_WARNING", log) 

def get_status_code(url):
    """ Perform HEAD request and return status code """
    try:
        request = Request(sanitize_url(url))
        request.add_header("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; "
                           "Windows NT 6.1; Trident/5.0)")
        request.get_method = lambda: 'HEAD'
        response = urlopen(request, context=ssl_unverified_context)
        # print response.info()
        return response.getcode()
    except HTTPError as e:
        return e.code
    except URLError as e:
        return e.reason
    except Exception as e:
        print(e, url)
        return None 

def http_request(uri, headers=None, data=None, method=None):
        uri, headers, data, method = prepare_request(uri, headers, data,
                                                     method)

        log.debug('Request %r with %r method' % (uri, method))
        req = http.Request(uri, headers=headers, data=data)
        req.get_method = lambda: method.upper()
        try:
            resp = http.urlopen(req)
            content = resp.read()
            resp.close()
            return resp, content
        except http.HTTPError as resp:
            content = resp.read()
            resp.close()
            return resp, content 

def _do_request(self, data):
        """
        使用 urllib 发送数据给服务器，如果发生错误会抛出异常。
        response的结果,会返回
        """
        encoded_data = urllib.urlencode(data).encode('utf8')
        try:
            request = urllib2.Request(self._debug_url_prefix, encoded_data)
            if not self._debug_write_data:      # 说明只检查,不真正写入数据
                request.add_header('Dry-Run', 'true')
            if self._request_timeout is not None:
                response = urllib2.urlopen(request, timeout=self._request_timeout)
            else:
                response = urllib2.urlopen(request)
        except urllib2.HTTPError as e:
            return e
        return response 

def test_wrong_exception_order(self):
        tmp = self.mkdtemp()
        path = os.path.join(tmp, 'xxx')
        self.write_file(path)
        dist_files = [('xxx', '2.6', path)]  # command, pyversion, filename
        self.write_file(self.rc, PYPIRC_LONG_PASSWORD)

        pkg_dir, dist = self.create_dist(dist_files=dist_files)
        tests = [
            (OSError('oserror'), 'oserror', OSError),
            (HTTPError('url', 400, 'httperror', {}, None),
             'Upload failed (400): httperror', DistutilsError),
        ]
        for exception, expected, raised_exception in tests:
            with self.subTest(exception=type(exception).__name__):
                with mock.patch('distutils.command.upload.urlopen',
                                new=mock.Mock(side_effect=exception)):
                    with self.assertRaises(raised_exception):
                        cmd = upload(dist)
                        cmd.ensure_finalized()
                        cmd.run()
                    results = self.get_logs(ERROR)
                    self.assertIn(expected, results[-1])
                    self.clear_logs() 

def test_wrong_exception_order(self):
        tmp = self.mkdtemp()
        path = os.path.join(tmp, 'xxx')
        self.write_file(path)
        dist_files = [('xxx', '2.6', path)]  # command, pyversion, filename
        self.write_file(self.rc, PYPIRC_LONG_PASSWORD)

        pkg_dir, dist = self.create_dist(dist_files=dist_files)
        tests = [
            (OSError('oserror'), 'oserror', OSError),
            (HTTPError('url', 400, 'httperror', {}, None),
             'Upload failed (400): httperror', DistutilsError),
        ]
        for exception, expected, raised_exception in tests:
            with self.subTest(exception=type(exception).__name__):
                with mock.patch('distutils.command.upload.urlopen',
                                new=mock.Mock(side_effect=exception)):
                    with self.assertRaises(raised_exception):
                        cmd = upload(dist)
                        cmd.ensure_finalized()
                        cmd.run()
                    results = self.get_logs(ERROR)
                    self.assertIn(expected, results[-1])
                    self.clear_logs() 

def _do_post(self, url, data, headers):
        try:  # Python 3
            if data != None:
                data = bytes(data, 'utf-8')
        except:  # Python 2
            pass

        try:
            req = url_request.Request(url=url, data=data, headers=headers)
            req.add_header('User-Agent', KKBOXHTTP.USER_AGENT)
            f = url_request.urlopen(req)
            r = f.read()
        except url_request.HTTPError as e:
            print(e.fp.read())
            raise (e)
        except Exception as e:
            raise (e)

        try:  # Python 3
            r = str(r, 'utf-8')
        except:  # Python 2
            pass
            
        json_object = json.loads(r)
        return json_object 

def HTTPcode(self):
		try:
			if self.agent == True:
				br = Browser()

				UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0"
				header = {"User-Agent" : UserAgent}
				br.set_handle_robots(False)
				br.addheaders = [("User-agent", "Fifefox")]
				
				resp = br.open(self.target).code

			else:
				resp = u.urlopen(self.target).getcode()
	
			return(resp)
		except (u.HTTPError, u.URLError):
			return(404) 

def do_request(self, req):
        if DEBUG:
            print('requesting', req.get_method(), req.get_full_url())

        opener = self.build_opener()
        opener.add_handler(self._cookie_processor)
        try:
            self._response = opener.open(req)
        except HTTPError as e:
            self._response = e

        self.url = self._response.geturl()
        self.path = get_selector(Request(self.url))
        self.data = self._response.read()
        self.status = self._response.code
        self._forms = None
        self.form = None

        return self.get_response() 

def test_wrong_exception_order(self):
        tmp = self.mkdtemp()
        path = os.path.join(tmp, 'xxx')
        self.write_file(path)
        dist_files = [('xxx', '2.6', path)]  # command, pyversion, filename
        self.write_file(self.rc, PYPIRC_LONG_PASSWORD)

        pkg_dir, dist = self.create_dist(dist_files=dist_files)
        tests = [
            (OSError('oserror'), 'oserror', OSError),
            (HTTPError('url', 400, 'httperror', {}, None),
             'Upload failed (400): httperror', DistutilsError),
        ]
        for exception, expected, raised_exception in tests:
            with self.subTest(exception=type(exception).__name__):
                with mock.patch('distutils.command.upload.urlopen',
                                new=mock.Mock(side_effect=exception)):
                    with self.assertRaises(raised_exception):
                        cmd = upload(dist)
                        cmd.ensure_finalized()
                        cmd.run()
                    results = self.get_logs(ERROR)
                    self.assertIn(expected, results[-1])
                    self.clear_logs() 

def proxy_request(service_name, instance_name, path, body=None, headers=None, method='POST'):
    target = get_env("TSURU_TARGET").rstrip("/")
    token = get_env("TSURU_TOKEN")
    url = "{}/services/{}/proxy/{}?callback={}".format(target, service_name, instance_name,
                                                       path)
    request = Request(url)
    request.add_header("Authorization", "bearer " + token)
    request.get_method = lambda: method
    if body:
        try:
            request.add_data(body)
        except AttributeError:
            request.data = body.encode('utf-8')
    if headers:
        for key, value in headers.items():
            request.add_header(key, value)
    try:
        return urlopen(request)
    except HTTPError as error:
        return error
    except Exception:
        raise 

def test_wrong_exception_order(self):
        tmp = self.mkdtemp()
        path = os.path.join(tmp, 'xxx')
        self.write_file(path)
        dist_files = [('xxx', '2.6', path)]  # command, pyversion, filename
        self.write_file(self.rc, PYPIRC_LONG_PASSWORD)

        pkg_dir, dist = self.create_dist(dist_files=dist_files)
        tests = [
            (OSError('oserror'), 'oserror', OSError),
            (HTTPError('url', 400, 'httperror', {}, None),
             'Upload failed (400): httperror', DistutilsError),
        ]
        for exception, expected, raised_exception in tests:
            with self.subTest(exception=type(exception).__name__):
                with mock.patch('distutils.command.upload.urlopen',
                                new=mock.Mock(side_effect=exception)):
                    with self.assertRaises(raised_exception):
                        cmd = upload(dist)
                        cmd.ensure_finalized()
                        cmd.run()
                    results = self.get_logs(ERROR)
                    self.assertIn(expected, results[-1])
                    self.clear_logs() 

def __init__(self, msg):
        status = '500 Internal Server Error'
        headers = { 'Content-Type': "text/html" }
        data = "<h1>Permission Error</h1><p>%s</p>" % msg
        if "headers" not in web.ctx: web.ctx.headers = []
        web.HTTPError.__init__(self, status, headers, data) 

def access_url(url):
    """Return list of lines in url; if HTTPError, repeat."""
    for line in urlopen(url).readlines():
        yield line.rstrip() 

def download_file(url, download_directory):
        """Download a remote file

        Args:
            download_directory: (string)

        Returns:
            (string) that path of the file that was just downloaded. If something failed during
                download, return None

        Raises:
            DownloadError
        """
        Output.print_information("Downloading " + url + " ...")

        parsed_url = urlparse(url)
        if parsed_url.path in ["/", ""]:
            file_name = parsed_url.netloc
        else:
            file_name = parsed_url.path.split("/")[-1]
        download_path = abspath(join(download_directory, file_name))

        try:
            with open(download_path, 'wb') as file_object:
                file_object.write(urlopen(url).read())
                return download_path

        except HTTPError as expn:
            raise DownloadError("HTTP error code " + str(expn.code) + " while retrieving " \
             + url + "\n" + str(expn.reason))
        except URLError as expn:
            raise DownloadError("HTTP URL error while retrieving " + url + "\n" + str(expn.reason))
        except Exception as expn:
            raise DownloadError("Unable to retrieve " + url + "\n" + str(expn)) 

def __init__(self, msg):
        status = '500 Internal Server Error'
        headers = { 'Content-Type': "text/html" }
        data = "<h1>Error</h1><p>%s</p>" % msg
        if "headers" not in web.ctx: web.ctx.headers = []
        web.HTTPError.__init__(self, status, headers, data) 

def post_data(url, data, contenttype = "", httpmethod = "", headers = {}):
    """
    Posts data (str or bytes) to a URL as the body
    httpmethod: POST by default.
    Returns dict of requestheaders (dict), requestbody (bytes), headers (str), response (str) and status (int)
    """
    # PYTHON3
    # Separate implementations here due to change in HTTPMessage response object
    # between 2 and 3. (.headers disappears to be replaced with as_string() due to new superclass)
    if sys.version_info[0] > 2:
        try:
            if contenttype != "": headers["Content-Type"] = contenttype
            if isinstance(data, str): data = str2bytes(data)
            req = urllib2.Request(url, data, headers)
            if httpmethod != "": req.get_method = lambda: httpmethod
            resp = urllib2.urlopen(req)
            return { "requestheaders": headers, "requestbody": data, "headers": resp.info().as_string(), "response": bytes2str(resp.read()), "status": resp.getcode() }
        except urllib2.HTTPError as e:
            return { "requestheaders": headers, "requestbody": data, "headers": e.info().as_string(), "response": bytes2str(e.read()), "status": e.getcode() }
    # PYTHON2
    else:
        try:
            if contenttype != "": headers["Content-Type"] = contenttype
            req = urllib2.Request(url, data, headers)
            if httpmethod != "": req.get_method = lambda: httpmethod
            resp = urllib2.urlopen(req)
            return { "requestheaders": headers, "requestbody": data, "headers": resp.info().headers, "response": encode_html(cunicode(resp.read())), "status": resp.getcode() }
        except urllib2.HTTPError as e:
            return { "requestheaders": headers, "requestbody": data, "headers": e.info().headers, "response": encode_html(cunicode(e.read())), "status": e.getcode() } 

def _make_api_query(dtype, includes=None, criteria=None, attributes=None,
                    suffix=None, returns='msg', verbose=False):
    """
    """

    url = 'https://api.brain-map.org/api/v2/data/{}/query.json?'.format(dtype)

    params = [includes, criteria, attributes]
    for key, value in zip(['include', 'criteria', 'only'], params):
        if value is not None:
            if isinstance(value, list):
                value = ','.join(value)
            url += '{}={}&'.format(key, urllib.parse.quote_plus(value))

    if suffix is not None:
        url += suffix

    if verbose:
        print("Querying {}...".format(url))
    response = urlopen(url)
    if response.status != 200:
        raise HTTPError('Failed to query API with code {}: {}'
                        .format(response.status, response.reason))

    info = json.loads(response.read().decode('utf-8'))

    if not info['success']:
        raise ValueError('Provided query {} is invalid. Please check '
                         'parameters and try again.'
                         .format(urllib.parse.unquote_plus(url)))
    elif info['total_rows'] == 0:
        raise ValueError('Provided query {} returned no results. Please '
                         'check parameters and try again.'
                         .format(urllib.parse.unquote_plus(url)))

    if returns is not None:
        info = info.get(returns, [])

    return info 

def getBestServer(servers):
    """Perform a speedtest.net latency request to determine which
    speedtest.net server has the lowest latency
    """

    results = {}
    for server in servers:
        cum = []
        url = '%s/latency.txt' % os.path.dirname(server['url'])
        urlparts = urlparse(url)
        for i in range(0, 3):
            try:
                if urlparts[0] == 'https':
                    h = HTTPSConnection(urlparts[1])
                else:
                    h = HTTPConnection(urlparts[1])
                headers = {'User-Agent': user_agent}
                start = timeit.default_timer()
                h.request("GET", urlparts[2], headers=headers)
                r = h.getresponse()
                total = (timeit.default_timer() - start)
            except (HTTPError, URLError, socket.error):
                cum.append(3600)
                continue
            text = r.read(9)
            if int(r.status) == 200 and text == 'test=test'.encode():
                cum.append(total)
            else:
                cum.append(3600)
            h.close()
        avg = round((sum(cum) / 6) * 1000, 3)
        results[avg] = server
    fastest = sorted(results.keys())[0]
    best = results[fastest]
    best['latency'] = fastest

    return best 

def catch_request(request):
    """Helper function to catch common exceptions encountered when
    establishing a connection with a HTTP/HTTPS request

    """

    try:
        uh = urlopen(request)
        return uh, False
    except (HTTPError, URLError, socket.error):
        e = sys.exc_info()[1]
        return None, e 

def send_tx(tx):
    s = io.BytesIO()
    tx.stream(s)
    tx_as_hex = b2h(s.getvalue())
    data = urlencode(dict(tx=tx_as_hex)).encode("utf8")
    URL = "http://blockchain.info/pushtx"
    try:
        d = urlopen(URL, data=data).read()
        return d
    except HTTPError as ex:
        d = ex.read()
        print(ex) 

def save_url(url, output_path):
    """Save a URL to disk."""
    valid_types = ['application/zip', 'application/octet-stream']
    r = None
    for _ in range(5):
        try:
            r = request.urlopen(url)
            break
        except request.HTTPError as e:
            reason = "None given"
            if e.reason:
                reason = e.reason
            arcpy.AddError("Unable to access '{}', (reason: {}).".format(
                url, reason))
        except request.URLError as e:
            arcpy.AddWarning("Access failed, trying again.")
            # retry all URLErrors
            time.sleep(3)

    if r and r.headers['content-type'] in valid_types and r.code == 200:
        arcpy.AddMessage("Saving URL to '{}'".format(output_path))
        with open(output_path, 'wb') as f:
            f.write(r.read())
    else:
        arcpy.AddError("Unable to access '{}', invalid content.".format(url))
        if r:
            arcpy.AddError("Content type: {}, response code: {}".format(
                r.headers['content-type'], r.code))
        msg = "Either a connectivity issue or restrictions on downloading " + \
              "prevented the tool from downloading. Please download the " + \
              "zip manually from {}".format(latest_url) + " and move it to " + \
              "the same location as this toolbox."
        arcpy.AddError(msg) 

def _do_request(self, data):
        """
        使用 urllib 发送数据给服务器，如果发生错误会抛出异常。
        """
        encoded_data = urllib.urlencode(data).encode('utf8')
        try:
            request = urllib2.Request(self._url_prefix, encoded_data)

            if self._request_timeout is not None:
                urllib2.urlopen(request, timeout=self._request_timeout)
            else:
                urllib2.urlopen(request)
        except urllib2.HTTPError as e:
            raise SensorsAnalyticsNetworkException(e)
        return True 

def _InvokeGitkitApi(self, method, params=None, need_service_account=True):
    """Invokes Gitkit API, with optional access token for service account.

    Args:
      method: string, the api method name.
      params: dict of optional parameters for the API.
      need_service_account: false if service account is not needed.

    Raises:
      GitkitClientError: if the request is bad.
      GitkitServerError: if Gitkit can not handle the request.

    Returns:
      API response as dict.
    """
    body = simplejson.dumps(params) if params else None
    req = urllib_request.Request(self.google_api_url + method)
    req.add_header('Content-type', 'application/json')
    if need_service_account:
      if self.credentials:
        access_token = self.credentials.get_access_token().access_token
      elif self.service_account_email and self.service_account_key:
        access_token = self._GetAccessToken()
      else:
        raise errors.GitkitClientError('Missing service account credentials')
      req.add_header('Authorization', 'Bearer ' + access_token)
    try:
      binary_body = body.encode('utf-8') if body else None
      raw_response = urllib_request.urlopen(req, binary_body).read()
    except urllib_request.HTTPError as err:
      if err.code == 400:
        raw_response = err.read()
      else:
        raise
    return self._CheckGitkitError(raw_response) 

def test_502(self):
        server = HttpTestServer('GET', '502', 'eRrOr')
        try:
            http_get('http://localhost:%d/path' % (server.port,))
        except HTTPError as e:
            self.assertTrue(isinstance(e, request.HTTPError))
            self.assertEqual(e.code, 502)
            self.assertDataEqual(e.response, 'eRrOr')
        else:
            self.assertFalse(True)
        server.join()