Python urllib3.exceptions.HTTPError() Examples

def commit_repository(self, branch):
        """
        :param branch: branch to be persisted
        Finalize the repository's scanning in bridgecrew's platform.
        """
        request = None
        try:
            request = http.request("PUT", f"{self.integrations_api_url}",
                                   body=json.dumps({"path": self.repo_path, "branch": branch}),
                                   headers={"Authorization": self.bc_api_key, "Content-Type": "application/json"})
            response = json.loads(request.data.decode("utf8"))
        except HTTPError as e:
            logging.error(f"Failed to commit repository {self.repo_path}\n{e}")
            raise e
        except JSONDecodeError as e:
            logging.error(f"Response of {self.integrations_api_url} is not a valid JSON\n{e}")
            raise e
        finally:
            if request.status == 201 and response["result"] == "Success":
                logging.info(f"Finalize repository {self.repo_id} in bridgecrew's platform")
            else:
                raise Exception(f"Failed to finalize repository {self.repo_id} in bridgecrew's platform\n{response}") 

def test_network_error_exception_detector(self):
        http_error = HTTPError()
        self.assertTrue(is_network_error(http_error))

        location_value_error = LocationValueError()
        self.assertTrue(is_network_error(location_value_error))

        pool_error = PoolError(None, 'an error')
        self.assertTrue(is_network_error(pool_error))

        exception = Exception()
        self.assertFalse(is_network_error(exception))

        w3_conn_error = Web3ConnectionException()
        self.assertFalse(is_network_error(w3_conn_error))

        setattr(w3_conn_error, 'errno', errno.ECONNABORTED)
        self.assertTrue(is_network_error(w3_conn_error))

        setattr(w3_conn_error, 'errno', errno.EPERM)
        self.assertFalse(is_network_error(w3_conn_error)) 

def _register(self):
        logger.info("Agent is starting.")
        try:
            agent_state = self.get_state()
            if agent_state.status == V1Statuses.STOPPED:
                logger.info(
                    "Agent has been stopped from the platform,"
                    "but the deployment is still running."
                    "Please either set the agent to starting or teardown the agent deployment."
                )
                return
            self.sync()
            self.log_agent_running()
        except (ApiException, HTTPError) as e:
            self.log_agent_failed(
                message="Could not start the agent {}.".format(repr(e))
            )
            sys.exit(1)
        atexit.register(self._wait) 

def artifacts(ctx):
    """Download outputs/artifacts for run.

    Uses /docs/core/cli/#caching

    Examples:

    \b
    $ polyaxon ops -uid=8aac02e3a62a4f0aaa257c59da5eab80 artifacts
    """
    owner, project_name, run_uuid = get_project_run_or_local(
        ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True,
    )
    try:
        client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid)
        client.download_artifacts()
    except (ApiException, HTTPError) as e:
        handle_cli_error(
            e, message="Could not download outputs for run `{}`.".format(run_uuid)
        )
        sys.exit(1)
    Printer.print_success("Files downloaded.") 

def get_json(self, url):
        try:
            response = self._http_client.request("GET", url, timeout=self.timeout)
            body = response.data
        except HTTPError as e:
            raise GeneralError("Unexpected error %s" % str(e))
        except socket.error as e:
            raise GeneralError("Socket Error: %s" % str(e))

        if response.status != 200:
            raise GeneralError("Server returned unexpected status code - {} - {}".format(response.status,
                                                                                         response.data))
        try:
            result = json.loads(body.decode('utf-8'))
        except Exception as e:
            # Error is parsing json
            raise GeneralError("Error parsing server response (%d) - %s. Got - %s" % (response.status, body, e))

        return result 

def wake_up(self):
        """ mod """
        global a_vin
        global a_display_name
        """ end mod """
        """ Request wake up of car. """
        delay = 1
        while True:
            try:
                result = self.vehicle.wake_up()["response"]
                logger.info("wake_up")
                for element in sorted(result):
                    logger.debug("   %s=%s" % (element, result[element]))
                    """ mod """
                    if element == "vin":
                        a_vin = result[element]
                    if element == "display_name":
                        a_display_name = result[element]
                return
            except (KeyError, HTTPError, URLError) as details:
                delay *= 2
                logger.warning("HTTP Error:" + str(details))
                logger.info("Waiting %d seconds before retrying." % delay)
                time.sleep(delay) 

def get(ctx):
    """Get info for current project, by project_name, or owner/project_name.

    Uses /docs/core/cli/#caching

    Examples:

    To get current project:

    \b
    $ polyaxon project get

    To get a project by name

    \b
    $ polyaxon project get owner/project
    """
    owner, project_name = get_project_or_local(ctx.obj.get("project"), is_cli=True)

    try:
        polyaxon_client = ProjectClient(owner=owner, project=project_name)
        polyaxon_client.refresh_data()
        config = polyaxon_client.client.sanitize_for_serialization(
            polyaxon_client.project_data
        )
        cache.cache(
            config_manager=ProjectManager,
            config=config,
            owner=owner,
            project=project_name,
        )
    except (ApiException, HTTPError) as e:
        handle_cli_error(e, message="Could not get project `{}`.".format(project_name))
        sys.exit(1)

    get_project_details(polyaxon_client.project_data) 

def catch_kubernetes_errors(func):
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except k8s_client.rest.ApiException as e:
            if e.status == 403:
                logger.exception('Permission denied')
            elif e.status != 409:  # Object exists or conflict in resource_version
                logger.exception('Unexpected error from Kubernetes API')
            return False
        except (RetryFailedError, HTTPException, HTTPError, socket.error, socket.timeout):
            return False
    return wrapper 

def catch_consul_errors(func):
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except (RetryFailedError, ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
            return False
    return wrapper 

def _do_http_request(self, retry, machines_cache, request_executor, method, path, fields=None, **kwargs):
        some_request_failed = False
        for i, base_uri in enumerate(machines_cache):
            if i > 0:
                logger.info("Retrying on %s", base_uri)
            try:
                response = request_executor(method, base_uri + path, fields=fields, **kwargs)
                response.data.decode('utf-8')
                self._check_cluster_id(response)
                if some_request_failed:
                    self.set_base_uri(base_uri)
                    self._refresh_machines_cache()
                return response
            except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
                self.http.clear()
                # switch to the next etcd node because we don't know exactly what happened,
                # whether the key didn't received an update or there is a network problem.
                if not retry and i + 1 < len(machines_cache):
                    self.set_base_uri(machines_cache[i + 1])
                if (isinstance(fields, dict) and fields.get("wait") == "true" and
                        isinstance(e, (ReadTimeoutError, ProtocolError))):
                    logger.debug("Watch timed out.")
                    raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
                logger.error("Request to server %s failed: %r", base_uri, e)
                logger.info("Reconnection allowed, looking for another server.")
                if not retry:
                    raise etcd.EtcdException('{0} {1} request failed'.format(method, path))
                some_request_failed = True

        raise etcd.EtcdConnectionFailed('No more machines in the cluster') 

def get_server_versions(polyaxon_client=None):
    polyaxon_client = polyaxon_client or PolyaxonClient()
    try:
        return polyaxon_client.versions_v1.get_versions()
    except ApiException as e:
        if e.status == 403:
            session_expired()
            sys.exit(1)
        handle_cli_error(e, message="Could not get cli version.")
        sys.exit(1)
    except HTTPError:
        Printer.print_error("Could not connect to remote server.")
        sys.exit(1) 

def get_log_handler(polyaxon_client=None):
    polyaxon_client = polyaxon_client or PolyaxonClient()
    try:
        return polyaxon_client.versions_v1.get_log_handler()
    except ApiException as e:
        if e.status == 403:
            session_expired()
            sys.exit(1)
        handle_cli_error(e, message="Could not get cli version.")
        sys.exit(1)
    except HTTPError:
        Printer.print_error("Could not connect to remote server.")
        sys.exit(1) 

def run(
    ctx,
    name: str,
    owner: str,
    project_name: str,
    description: str,
    tags: List[str],
    op_spec: V1Operation,
    log: bool,
    watch: bool,
):
    def create_run():
        click.echo("Creating a run.")
        try:
            polyaxon_client = RunClient(owner=owner, project=project_name)
            response = polyaxon_client.create(
                name=name, description=description, tags=tags, content=op_spec
            )
            config = polyaxon_client.client.sanitize_for_serialization(response)
            cache.cache(
                config_manager=RunManager,
                config=config,
                owner=owner,
                project=project_name,
            )
            Printer.print_success("A new run `{}` was created".format(response.uuid))
        except (ApiException, HTTPError) as e:
            handle_cli_error(e, message="Could not create a run.")
            sys.exit(1)

    create_run()

    # Check if we need to invoke logs
    if watch:
        ctx.obj = {"project": "{}/{}".format(owner, project_name)}
        ctx.invoke(statuses, watch=True)

    # Check if we need to invoke logs
    if log:
        ctx.obj = {"project": "{}/{}".format(owner, project_name)}
        ctx.invoke(run_logs) 

def delete(ctx):
    """Delete a run.

    Uses /docs/core/cli/#caching

    Example:

    \b
    $ polyaxon ops delete

    \b
    $ polyaxon ops --uid=8aac02e3a62a4f0aaa257c59da5eab80 delete  # project is cached

    \b
    $ polyaxon ops --project=cats-vs-dogs -uid 8aac02e3a62a4f0aaa257c59da5eab80 delete
    """
    owner, project_name, run_uuid = get_project_run_or_local(
        ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True,
    )
    if not click.confirm("Are sure you want to delete run `{}`".format(run_uuid)):
        click.echo("Existing without deleting the run.")
        sys.exit(1)

    try:
        polyaxon_client = RunClient(
            owner=owner, project=project_name, run_uuid=run_uuid
        )
        polyaxon_client.delete()
        # Purge caching
        RunManager.purge()
    except (ApiException, HTTPError) as e:
        handle_cli_error(e, message="Could not delete run `{}`.".format(run_uuid))
        sys.exit(1)

    Printer.print_success("Run `{}` was delete successfully".format(run_uuid)) 

def stop(ctx, yes):
    """Stop run.

    Uses /docs/core/cli/#caching

    Examples:

    \b
    $ polyaxon ops stop

    \b
    $ polyaxon ops --uid=8aac02e3a62a4f0aaa257c59da5eab80 stop
    """
    owner, project_name, run_uuid = get_project_run_or_local(
        ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True,
    )
    if not yes and not click.confirm(
        "Are sure you want to stop " "run `{}`".format(run_uuid)
    ):
        click.echo("Existing without stopping run.")
        sys.exit(0)

    try:
        polyaxon_client = RunClient(
            owner=owner, project=project_name, run_uuid=run_uuid
        )
        polyaxon_client.stop()
    except (ApiException, HTTPError) as e:
        handle_cli_error(e, message="Could not stop run `{}`.".format(run_uuid))
        sys.exit(1)

    Printer.print_success("Run is being stopped.") 

def restart(ctx, copy, polyaxonfile):
    """Restart run.

    Uses /docs/core/cli/#caching

    Examples:

    \b
    $ polyaxon run --uid=8aac02e3a62a4f0aaa257c59da5eab80 restart
    """
    content = None
    if polyaxonfile:
        content = "{}".format(ConfigSpec.read_from(polyaxonfile))

    owner, project_name, run_uuid = get_project_run_or_local(
        ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True,
    )
    try:
        polyaxon_client = RunClient(
            owner=owner, project=project_name, run_uuid=run_uuid
        )
        response = polyaxon_client.restart(override_config=content, copy=copy)
        Printer.print_success(
            "Run was {} with uid {}".format(
                "copied" if copy else "restarted", response.uuid
            )
        )
    except (ApiException, HTTPError) as e:
        handle_cli_error(e, message="Could not restart run `{}`.".format(run_uuid))
        sys.exit(1) 

def resume(ctx, polyaxonfile):
    """Resume run.

    Uses /docs/core/cli/#caching

    Examples:

    \b
    $ polyaxon ops --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume
    """
    content = None
    if polyaxonfile:
        content = "{}".format(ConfigSpec.read_from(polyaxonfile))

    owner, project_name, run_uuid = get_project_run_or_local(
        ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True,
    )
    try:
        polyaxon_client = RunClient(
            owner=owner, project=project_name, run_uuid=run_uuid
        )
        response = polyaxon_client.resume(override_config=content)
        Printer.print_success("Run was resumed with uid {}".format(response.uuid))
    except (ApiException, HTTPError) as e:
        handle_cli_error(e, message="Could not resume run `{}`.".format(run_uuid))
        sys.exit(1) 

def create(ctx, name, owner, description, tags, private, init):
    """Create a new project.

    Uses /docs/core/cli/#caching

    Example:

    \b
    $ polyaxon project create --name=cats-vs-dogs --description="Image Classification with DL"
    """
    owner = owner or AuthConfigManager.get_value("username") or DEFAULT
    tags = validate_tags(tags)

    if not owner:
        Printer.print_error(
            "Please login first or provide a valid owner --owner. "
            "`polyaxon login --help`"
        )
        sys.exit(1)

    try:
        project_config = V1Project(
            name=name, description=description, tags=tags, is_public=not private
        )
        polyaxon_client = ProjectClient(owner=owner)
        _project = polyaxon_client.create(project_config)
        config = polyaxon_client.client.sanitize_for_serialization(_project)
        cache.cache(config_manager=ProjectManager, config=config)
    except (ApiException, HTTPError) as e:
        handle_cli_error(e, message="Could not create project `{}`.".format(name))
        sys.exit(1)

    Printer.print_success(
        "Project `{}` was created successfully.".format(_project.name)
    )

    if init:
        ctx.obj = {}
        ctx.invoke(init_project, project=name) 

def impersonate(owner: str, project: str, run_uuid: str, client: PolyaxonClient = None):
    try:
        client = client or PolyaxonClient()
        response = client.runs_v1.impersonate_token(owner, project, run_uuid)
        polyaxon_client = PolyaxonClient(token=response.token)
        user = polyaxon_client.users_v1.get_user()
        access_token = AccessTokenConfig(username=user.username, token=response.token)
        create_context_auth(access_token)
    except (ApiException, HTTPError) as e:
        raise PolyaxonClientException(
            "This worker is not allowed to run this job %s." % e
        ) 

def get_statuses(self, last_status: str = None):
        """Gets the run's statuses.

        [Run API](/docs/api/#operation/GetRunStatus)

        Args:
            last_status: str, a valid [Statuses](/docs/core/specification/lifecycle/) value.

        Returns:
            Tuple[str, List[Conditions]], last status and ordered status conditions.
        """
        try:
            response = self.client.runs_v1.get_run_statuses(
                self.owner, self.project, self.run_uuid
            )
            if not last_status:
                return response.status, response.status_conditions
            if last_status == response.status:
                return last_status, []

            _conditions = []
            for c in reversed(response.status_conditions):
                if c.type == last_status:
                    break
                _conditions.append(c)

            return response.status, reversed(_conditions)

        except (ApiException, HTTPError) as e:
            raise PolyaxonClientException("Api error: %s" % e) from e 

def download(self):
        self.create_core_agent_dir()
        self.obtain_download_lock()
        if self.download_lock_fd is not None:
            try:
                downloaded = self.download_package()
                if downloaded:
                    self.untar()
            except (OSError, HTTPError):
                logger.exception("Exception raised while downloading Core Agent")
            finally:
                self.release_download_lock() 

def setup_bridgecrew_credentials(self, bc_api_key, repo_id):
        """
        Setup credentials against Bridgecrew's platform.
        :param repo_id: Identity string of the scanned repository, of the form <repo_owner>/<repo_name>
        :param bc_api_key: Bridgecrew issued API key
        """
        self.bc_api_key = bc_api_key
        self.repo_id = repo_id
        try:
            request = http.request("POST", self.integrations_api_url, body=json.dumps({"repoId": repo_id}),
                                   headers={"Authorization": bc_api_key, "Content-Type": "application/json"})
            response = json.loads(request.data.decode("utf8"))
            if 'Message' in response:
                if response['Message'] == UNAUTHORIZED_MESSAGE:
                    raise BridgecrewAuthError()
            repo_full_path = response["path"]
            self.bucket, self.repo_path = repo_full_path.split("/", 1)
            self.timestamp = self.repo_path.split("/")[-1]
            self.credentials = response["creds"]
            self.s3_client = boto3.client("s3",
                                          aws_access_key_id=self.credentials["AccessKeyId"],
                                          aws_secret_access_key=self.credentials["SecretAccessKey"],
                                          aws_session_token=self.credentials["SessionToken"],
                                          region_name=DEFAULT_REGION
                                          )
            sleep(10)  # Wait for the policy to update
        except HTTPError as e:
            logging.error(f"Failed to get customer assumed role\n{e}")
            raise e
        except ClientError as e:
            logging.error(f"Failed to initiate client with credentials {self.credentials}\n{e}")
            raise e
        except JSONDecodeError as e:
            logging.error(f"Response of {self.integrations_api_url} is not a valid JSON\n{e}")
            raise e 

def is_network_error(exception) -> bool:
    """
    :param exception: an exception error instance
    :return: True if exception detected as a network error, False otherwise
    """
    network_errors = [errno.ECONNABORTED, errno.ECONNREFUSED, errno.ENETRESET, errno.ECONNRESET,
                      errno.ENETUNREACH, errno.ENETDOWN]

    if isinstance(exception, HTTPError) or isinstance(exception, RequestException) or \
            hasattr(exception, 'errno') and exception.errno in network_errors:
        return True

    return False 

def validated_json_payload(response):
    """Asserts that the HTTP response was successful and valid JSON."""
    if response.status != 200:
        raise HTTPError(response.status, "non-200 response")

    try:
        data = response.data.decode('utf-8')
        payload = json.loads(data)
    except Exception as e:
        raise Exception("JSON error %s: %s" % (str(e), data[0:1024]))

    return payload 

def test_broken_response(self):
        request = self.factory.get('/')

        urlopen_mock = MagicMock(side_effect=HTTPError())
        with patch(URLOPEN, urlopen_mock), self.assertRaises(HTTPError):
            CustomProxyView.as_view()(request, path='/') 

def retrieve(self):
        """Retrieve credential value and its expiry from IAM EC2."""
        # Get role names.
        creds_path = "/latest/meta-data/iam/security-credentials"
        url = self._endpoint + creds_path
        res = self._http_client.urlopen("GET", url)
        if res.status != 200:
            raise HTTPError(
                "request failed with status {0}".format(res.status),
            )
        role_names = res.data.decode("utf-8").split("\n")
        if not role_names:
            raise ResponseError("no role names found in response")

        # Get credentials of first role.
        url = self._endpoint + creds_path + "/" + role_names[0]
        res = self._http_client.urlopen("GET", url)
        if res.status != 200:
            raise HTTPError(
                "request failed with status {0}".format(res.status),
            )
        data = json.loads(res.data)
        if data["Code"] != "Success":
            raise ResponseError(
                "credential retrieval failed with code {0}".format(
                    data["Code"]),
            )

        try:
            expiration = datetime.strptime(data["Expiration"], RFC3339NANO)
        except ValueError:
            expiration = datetime.strptime(data["Expiration"], RFC3339)
        return Value(
            data["AccessKeyId"],
            data["SecretAccessKey"],
            session_token=data["Token"],
        ), expiration + timedelta(minutes=5) 

def dataset_add_remote_file(
    cache, user, user_job_id, project_id, create_dataset, commit_message,
    short_name, url
):
    """Add a remote file to a specified dataset."""
    user = cache.ensure_user(user)
    user_job = cache.get_job(user, user_job_id)
    project = cache.get_project(user, project_id)

    try:
        user_job.in_progress()

        with chdir(project.abs_path):
            urls = url if isinstance(url, list) else [url]
            add_file(
                urls,
                short_name,
                create=create_dataset,
                commit_message=commit_message
            )

            _, remote_branch = repo_sync(
                Repo(project.abs_path), remote='origin'
            )
            user_job.update_extras('remote_branch', remote_branch)

            user_job.complete()
    except (HTTPError, BaseException, GitCommandError) as e:
        user_job.fail_job(str(e)) 

def dataset_import(
    cache,
    user,
    user_job_id,
    project_id,
    dataset_uri,
    short_name=None,
    extract=False,
    timeout=None,
):
    """Job for dataset import."""
    user = cache.ensure_user(user)
    user_job = cache.get_job(user, user_job_id)
    project = cache.get_project(user, project_id)

    with chdir(project.abs_path):
        try:
            user_job.in_progress()

            import_dataset(
                dataset_uri,
                short_name,
                extract,
                commit_message=f'service: dataset import {dataset_uri}',
                progress=DatasetImportJobProcess(cache, user_job)
            )

            _, remote_branch = repo_sync(
                Repo(project.abs_path), remote='origin'
            )
            user_job.update_extras('remote_branch', remote_branch)

            user_job.complete()
        except (
            HTTPError, ParameterError, DatasetExistsError, GitCommandError
        ) as exp:
            user_job.fail_job(str(exp))

            # Reraise exception, so we see trace in job metadata.
            raise exp 

def retry(func):
        def wrapped(*args, **kwargs):
            count = 0
            while True:
                try:
                    return func(*args, **kwargs)
                except (HTTPException, HTTPError, socket.error, socket.timeout):
                    if count >= 10:
                        raise
                    logger.info('Throttling API requests...')
                    time.sleep(2 ** count * 0.5)
                    count += 1
        return wrapped 

def __init__(self, config):
        super(Consul, self).__init__(config)
        self._scope = config['scope']
        self._session = None
        self.__do_not_watch = False
        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
                            retry_exceptions=(ConsulInternalError, HTTPException,
                                              HTTPError, socket.error, socket.timeout))

        kwargs = {}
        if 'url' in config:
            r = urlparse(config['url'])
            config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500})
        elif 'host' in config:
            host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500)
            config['host'] = host
            if 'port' not in config:
                config['port'] = int(port)

        if config.get('cacert'):
            config['ca_cert'] = config.pop('cacert')

        if config.get('key') and config.get('cert'):
            config['cert'] = (config['cert'], config['key'])

        config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc', 'consistency')
        kwargs = {p: config.get(p) for p in config_keys if config.get(p)}

        verify = config.get('verify')
        if not isinstance(verify, bool):
            verify = parse_bool(verify)
        if isinstance(verify, bool):
            kwargs['verify'] = verify

        self._client = ConsulClient(**kwargs)
        self.set_retry_timeout(config['retry_timeout'])
        self.set_ttl(config.get('ttl') or 30)
        self._last_session_refresh = 0
        self.__session_checks = config.get('checks', [])
        self._register_service = config.get('register_service', False)
        if self._register_service:
            self._service_name = service_name_from_scope_name(self._scope)
            if self._scope != self._service_name:
                logger.warning('Using %s as consul service name instead of scope name %s', self._service_name,
                               self._scope)
        self._service_check_interval = config.get('service_check_interval', '5s')
        if not self._ctl:
            self.create_session()