Python Examples of six.moves.http

Source File: cloudstack.py From cloudbase-init with Apache License 2.0

6 votes

def _password_client(self, body=None, headers=None, decode=True):
        """Client for the Password Server."""
        port = CONF.cloudstack.password_server_port
        with contextlib.closing(http_client.HTTPConnection(
                self._metadata_host, port, timeout=TIMEOUT)) as connection:
            try:
                connection.request("GET", "/", body=body, headers=headers)
                response = connection.getresponse()
            except http_client.HTTPException as exc:
                LOG.error("Request failed: %s", exc)
                raise

            content = response.read()
            if decode:
                content = encoding.get_as_string(content)

            if response.status != 200:
                raise http_client.HTTPException(
                    "%(status)s %(reason)s - %(message)r",
                    {"status": response.status, "reason": response.reason,
                     "message": content})

        return content

Source File: test_impersonated_credentials.py From google-auth-library-python with Apache License 2.0

6 votes

def test_refresh_failure_http_error(self, mock_donor_credentials):
        credentials = self.make_credentials(lifetime=None)

        response_body = {}

        request = self.make_request(
            data=json.dumps(response_body), status=http_client.HTTPException
        )

        with pytest.raises(exceptions.RefreshError) as excinfo:
            credentials.refresh(request)

        assert excinfo.match(impersonated_credentials._REFRESH_ERROR)

        assert not credentials.valid
        assert credentials.expired

Source File: gce.py From jarvis with GNU General Public License v2.0

6 votes

def _refresh(self, http):
        """Refreshes the access token.

        Skip all the storage hoops and just refresh using the API.

        Args:
            http: an object to be used to make HTTP requests.

        Raises:
            HttpAccessTokenRefreshError: When the refresh fails.
        """
        try:
            self._retrieve_info(http)
            self.access_token, self.token_expiry = _metadata.get_token(
                http, service_account=self.service_account_email)
        except http_client.HTTPException as err:
            raise client.HttpAccessTokenRefreshError(str(err))

Source File: kubernetes.py From patroni with MIT License

5 votes

def catch_kubernetes_errors(func):
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except k8s_client.rest.ApiException as e:
            if e.status == 403:
                logger.exception('Permission denied')
            elif e.status != 409:  # Object exists or conflict in resource_version
                logger.exception('Unexpected error from Kubernetes API')
            return False
        except (RetryFailedError, HTTPException, HTTPError, socket.error, socket.timeout):
            return False
    return wrapper

Source File: package_index.py From PhonePi_SampleServer with MIT License

5 votes

def open_url(self, url, warning=None):
        if url.startswith('file:'):
            return local_open(url)
        try:
            return open_with_auth(url, self.opener)
        except (ValueError, http_client.InvalidURL) as v:
            msg = ' '.join([str(arg) for arg in v.args])
            if warning:
                self.warn(warning, msg)
            else:
                raise DistutilsError('%s %s' % (url, msg))
        except urllib.error.HTTPError as v:
            return v
        except urllib.error.URLError as v:
            if warning:
                self.warn(warning, v.reason)
            else:
                raise DistutilsError("Download error for %s: %s"
                                     % (url, v.reason))
        except http_client.BadStatusLine as v:
            if warning:
                self.warn(warning, v.line)
            else:
                raise DistutilsError(
                    '%s returned a bad status line. The server might be '
                    'down, %s' %
                    (url, v.line)
                )
        except (http_client.HTTPException, socket.error) as v:
            if warning:
                self.warn(warning, v)
            else:
                raise DistutilsError("Download error for %s: %s"
                                     % (url, v))

Source File: test_docker_io.py From girder_worker with Apache License 2.0

5 votes

def test_ChunkedTransferEncodingStreamWriter_write_on_exception_still_closes():
    data = b'BOGUS DATA'
    with mock.patch('girder_worker.docker.io.httplib.HTTPConnection', autospec=True):
        s = ChunkedTransferEncodingStreamWriter('http://bogus.url.com/')
        s.conn.send.side_effect = httplib.HTTPException('Bogus Exception')
        with pytest.raises(httplib.HTTPException):
            s.write(data)
            s.conn.close.assert_called_once()
            assert s._closed is True

Source File: etcd.py From patroni with MIT License

5 votes

def _do_http_request(self, retry, machines_cache, request_executor, method, path, fields=None, **kwargs):
        some_request_failed = False
        for i, base_uri in enumerate(machines_cache):
            if i > 0:
                logger.info("Retrying on %s", base_uri)
            try:
                response = request_executor(method, base_uri + path, fields=fields, **kwargs)
                response.data.decode('utf-8')
                self._check_cluster_id(response)
                if some_request_failed:
                    self.set_base_uri(base_uri)
                    self._refresh_machines_cache()
                return response
            except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
                self.http.clear()
                # switch to the next etcd node because we don't know exactly what happened,
                # whether the key didn't received an update or there is a network problem.
                if not retry and i + 1 < len(machines_cache):
                    self.set_base_uri(machines_cache[i + 1])
                if (isinstance(fields, dict) and fields.get("wait") == "true" and
                        isinstance(e, (ReadTimeoutError, ProtocolError))):
                    logger.debug("Watch timed out.")
                    raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
                logger.error("Request to server %s failed: %r", base_uri, e)
                logger.info("Reconnection allowed, looking for another server.")
                if not retry:
                    raise etcd.EtcdException('{0} {1} request failed'.format(method, path))
                some_request_failed = True

        raise etcd.EtcdConnectionFailed('No more machines in the cluster')

Source File: consul.py From patroni with MIT License

5 votes

def catch_consul_errors(func):
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except (RetryFailedError, ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
            return False
    return wrapper

Source File: _metadata.py From jarvis with GNU General Public License v2.0

5 votes

def get(http, path, root=METADATA_ROOT, recursive=None):
    """Fetch a resource from the metadata server.

    Args:
        http: an object to be used to make HTTP requests.
        path: A string indicating the resource to retrieve. For example,
            'instance/service-accounts/defualt'
        root: A string indicating the full path to the metadata server root.
        recursive: A boolean indicating whether to do a recursive query of
            metadata. See
            https://cloud.google.com/compute/docs/metadata#aggcontents

    Returns:
        A dictionary if the metadata server returns JSON, otherwise a string.

    Raises:
        http_client.HTTPException if an error corrured while
        retrieving metadata.
    """
    url = urlparse.urljoin(root, path)
    url = _helpers._add_query_parameter(url, 'recursive', recursive)

    response, content = transport.request(
        http, url, headers=METADATA_HEADERS)

    if response.status == http_client.OK:
        decoded = _helpers._from_bytes(content)
        if response['content-type'] == 'application/json':
            return json.loads(decoded)
        else:
            return decoded
    else:
        raise http_client.HTTPException(
            'Failed to retrieve {0} from the Google Compute Engine'
            'metadata service. Response:\n{1}'.format(url, response))

Source File: publisher.py From PerfKitBenchmarker with Apache License 2.0

5 votes

def _Publish(self, formated_samples):
    try:
      self._CreateDB()
      body = '\n'.join(formated_samples)
      self._WriteData(body)
    except (IOError, httplib.HTTPException) as http_exception:
      logging.error('Error connecting to the database:  %s', http_exception)

Source File: _metadata.py From aqua-monitor with GNU Lesser General Public License v3.0

5 votes

def get(http, path, root=METADATA_ROOT, recursive=None):
    """Fetch a resource from the metadata server.

    Args:
        http: an object to be used to make HTTP requests.
        path: A string indicating the resource to retrieve. For example,
            'instance/service-accounts/default'
        root: A string indicating the full path to the metadata server root.
        recursive: A boolean indicating whether to do a recursive query of
            metadata. See
            https://cloud.google.com/compute/docs/metadata#aggcontents

    Returns:
        A dictionary if the metadata server returns JSON, otherwise a string.

    Raises:
        http_client.HTTPException if an error corrured while
        retrieving metadata.
    """
    url = urlparse.urljoin(root, path)
    url = _helpers._add_query_parameter(url, 'recursive', recursive)

    response, content = transport.request(
        http, url, headers=METADATA_HEADERS)

    if response.status == http_client.OK:
        decoded = _helpers._from_bytes(content)
        if response['content-type'] == 'application/json':
            return json.loads(decoded)
        else:
            return decoded
    else:
        raise http_client.HTTPException(
            'Failed to retrieve {0} from the Google Compute Engine'
            'metadata service. Response:\n{1}'.format(url, response))

Source File: http.py From pledgeservice with Apache License 2.0

5 votes

def check_server(host, port, path_info='/', timeout=3, retries=30):
    """Perform a request until the server reply"""
    if retries < 0:
        return 0
    conn = http_client.HTTPConnection(host, port, timeout=timeout)
    time.sleep(.3)
    for i in range(retries):
        try:
            conn.request('GET', path_info)
            res = conn.getresponse()
            return res.status
        except (socket.error, http_client.HTTPException):
            time.sleep(.3)
    return 0

Source File: callback_endpoint.py From spilo with Apache License 2.0

5 votes

def retry(func):
        def wrapped(*args, **kwargs):
            count = 0
            while True:
                try:
                    return func(*args, **kwargs)
                except (HTTPException, HTTPError, socket.error, socket.timeout):
                    if count >= 10:
                        raise
                    logger.info('Throttling API requests...')
                    time.sleep(2 ** count * 0.5)
                    count += 1
        return wrapped

Source File: package_index.py From Safejumper-for-Desktop with GNU General Public License v2.0

5 votes

def open_url(self, url, warning=None):
        if url.startswith('file:'):
            return local_open(url)
        try:
            return open_with_auth(url, self.opener)
        except (ValueError, http_client.InvalidURL) as v:
            msg = ' '.join([str(arg) for arg in v.args])
            if warning:
                self.warn(warning, msg)
            else:
                raise DistutilsError('%s %s' % (url, msg))
        except urllib.error.HTTPError as v:
            return v
        except urllib.error.URLError as v:
            if warning:
                self.warn(warning, v.reason)
            else:
                raise DistutilsError("Download error for %s: %s"
                                     % (url, v.reason))
        except http_client.BadStatusLine as v:
            if warning:
                self.warn(warning, v.line)
            else:
                raise DistutilsError(
                    '%s returned a bad status line. The server might be '
                    'down, %s' %
                    (url, v.line)
                )
        except (http_client.HTTPException, socket.error) as v:
            if warning:
                self.warn(warning, v)
            else:
                raise DistutilsError("Download error for %s: %s"
                                     % (url, v))

Source File: consul.py From patroni with MIT License

4 votes

def __init__(self, config):
        super(Consul, self).__init__(config)
        self._scope = config['scope']
        self._session = None
        self.__do_not_watch = False
        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
                            retry_exceptions=(ConsulInternalError, HTTPException,
                                              HTTPError, socket.error, socket.timeout))

        kwargs = {}
        if 'url' in config:
            r = urlparse(config['url'])
            config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500})
        elif 'host' in config:
            host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500)
            config['host'] = host
            if 'port' not in config:
                config['port'] = int(port)

        if config.get('cacert'):
            config['ca_cert'] = config.pop('cacert')

        if config.get('key') and config.get('cert'):
            config['cert'] = (config['cert'], config['key'])

        config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc', 'consistency')
        kwargs = {p: config.get(p) for p in config_keys if config.get(p)}

        verify = config.get('verify')
        if not isinstance(verify, bool):
            verify = parse_bool(verify)
        if isinstance(verify, bool):
            kwargs['verify'] = verify

        self._client = ConsulClient(**kwargs)
        self.set_retry_timeout(config['retry_timeout'])
        self.set_ttl(config.get('ttl') or 30)
        self._last_session_refresh = 0
        self.__session_checks = config.get('checks', [])
        self._register_service = config.get('register_service', False)
        if self._register_service:
            self._service_name = service_name_from_scope_name(self._scope)
            if self._scope != self._service_name:
                logger.warning('Using %s as consul service name instead of scope name %s', self._service_name,
                               self._scope)
        self._service_check_interval = config.get('service_check_interval', '5s')
        if not self._ctl:
            self.create_session()

Source File: kubernetes.py From patroni with MIT License

4 votes

def __init__(self, config):
        self._labels = config['labels']
        self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
        self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
        self._namespace = config.get('namespace') or 'default'
        self._role_label = config.get('role_label', 'role')
        config['namespace'] = ''
        super(Kubernetes, self).__init__(config)
        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
                            retry_exceptions=(KubernetesRetriableException, HTTPException,
                                              HTTPError, socket.error, socket.timeout))
        self._ttl = None
        try:
            k8s_config.load_incluster_config()
        except k8s_config.ConfigException:
            k8s_config.load_kube_config(context=config.get('context', 'local'))

        self.__my_pod = None
        self.__ips = [] if config.get('patronictl') else [config.get('pod_ip')]
        self.__ports = []
        for p in config.get('ports', [{}]):
            port = {'port': int(p.get('port', '5432'))}
            port.update({n: p[n] for n in ('name', 'protocol') if p.get(n)})
            self.__ports.append(k8s_client.V1EndpointPort(**port))

        self._api = CoreV1ApiProxy(config.get('use_endpoints'))
        self._should_create_config_service = self._api.use_endpoints
        self.reload_config(config)
        # leader_observed_record, leader_resource_version, and leader_observed_time are used only for leader race!
        self._leader_observed_record = {}
        self._leader_observed_time = None
        self._leader_resource_version = None
        self.__do_not_watch = False

        self._condition = Condition()

        pods_func = functools.partial(self._api.list_namespaced_pod, self._namespace,
                                      label_selector=self._label_selector)
        self._pods = ObjectCache(self, pods_func, self._retry, self._condition)

        kinds_func = functools.partial(self._api.list_namespaced_kind, self._namespace,
                                       label_selector=self._label_selector)
        self._kinds = ObjectCache(self, kinds_func, self._retry, self._condition, self._name)

Source File: _http_client.py From aws-kube-codesuite with Apache License 2.0

4 votes

def __call__(self, url, method='GET', body=None, headers=None,
                 timeout=None, **kwargs):
        """Make an HTTP request using http.client.

        Args:
            url (str): The URI to be requested.
            method (str): The HTTP method to use for the request. Defaults
                to 'GET'.
            body (bytes): The payload / body in HTTP request.
            headers (Mapping): Request headers.
            timeout (Optional(int)): The number of seconds to wait for a
                response from the server. If not specified or if None, the
                socket global default timeout will be used.
            kwargs: Additional arguments passed throught to the underlying
                :meth:`~http.client.HTTPConnection.request` method.

        Returns:
            Response: The HTTP response.

        Raises:
            google.auth.exceptions.TransportError: If any exception occurred.
        """
        # socket._GLOBAL_DEFAULT_TIMEOUT is the default in http.client.
        if timeout is None:
            timeout = socket._GLOBAL_DEFAULT_TIMEOUT

        # http.client doesn't allow None as the headers argument.
        if headers is None:
            headers = {}

        # http.client needs the host and path parts specified separately.
        parts = urllib.parse.urlsplit(url)
        path = urllib.parse.urlunsplit(
            ('', '', parts.path, parts.query, parts.fragment))

        if parts.scheme != 'http':
            raise exceptions.TransportError(
                'http.client transport only supports the http scheme, {}'
                'was specified'.format(parts.scheme))

        connection = http_client.HTTPConnection(parts.netloc, timeout=timeout)

        try:
            _LOGGER.debug('Making request: %s %s', method, url)

            connection.request(
                method, path, body=body, headers=headers, **kwargs)
            response = connection.getresponse()
            return Response(response)

        except (http_client.HTTPException, socket.error) as exc:
            raise exceptions.TransportError(exc)

        finally:
            connection.close()