Python botocore.exceptions.WaiterError() Examples

def retry(function, *args, **kwargs):
    """ Retries a function up to max_retries, waiting `timeout` seconds between tries.
        This function is designed to retry both boto3 and fabric calls.  In the
        case of boto3, it is necessary because sometimes aws calls return too
        early and a resource needed by the next call is not yet available. """
    max_retries = kwargs.pop("max_retries", 10)
    timeout = kwargs.pop("timeout", 1)
    for attempt in range(max_retries):
        try:
            ret = yield thread_pool.submit(function, *args, **kwargs)
            return ret
        except (ClientError, WaiterError, NetworkError, RemoteCmdExecutionError, EOFError, SSHException, ChannelException) as e:
            #EOFError can occur in fabric
            logger.error("Failure in %s with args %s and kwargs %s" % (function.__name__, args, kwargs))
            logger.info("retrying %s, (~%s seconds elapsed)" % (function.__name__, attempt * 3))
            yield gen.sleep(timeout)
    else:
        logger.error("Failure in %s with args %s and kwargs %s" % (function.__name__, args, kwargs))
        yield gen.sleep(0.1) #this line exists to allow the logger time to print
        return ("RETRY_FAILED")

#########################################################################################################
######################################################################################################### 

def test__wait_for_registration_waiter_fails_describe_fails(project):
    mock_cfn_client = MagicMock(
        spec=["describe_type_registration", "set_type_default_version", "get_waiter"]
    )
    mock_cfn_client.describe_type_registration.side_effect = ClientError(
        BLANK_CLIENT_ERROR, "DescribeTypeRegistration"
    )
    mock_waiter = MagicMock(spec=["wait"])
    mock_waiter.wait.side_effect = WaiterError(
        "TypeRegistrationComplete",
        "Waiter encountered a terminal failure state",
        DESCRIBE_TYPE_FAILED_RETURN,
    )

    mock_cfn_client.get_waiter.return_value = mock_waiter

    with pytest.raises(DownstreamError):
        project._wait_for_registration(mock_cfn_client, REGISTRATION_TOKEN, False)

    mock_cfn_client.describe_type_registration.assert_called_once_with(
        RegistrationToken=REGISTRATION_TOKEN
    )
    mock_cfn_client.set_type_default_version.assert_not_called()
    mock_waiter.wait.assert_called_once_with(RegistrationToken=REGISTRATION_TOKEN) 

def test__wait_for_registration_waiter_fails(project):
    mock_cfn_client = MagicMock(
        spec=["describe_type_registration", "set_type_default_version", "get_waiter"]
    )
    mock_cfn_client.describe_type_registration.return_value = (
        DESCRIBE_TYPE_FAILED_RETURN
    )
    mock_waiter = MagicMock(spec=["wait"])
    mock_waiter.wait.side_effect = WaiterError(
        "TypeRegistrationComplete",
        "Waiter encountered a terminal failure state",
        DESCRIBE_TYPE_FAILED_RETURN,
    )
    mock_cfn_client.get_waiter.return_value = mock_waiter

    with pytest.raises(DownstreamError):
        project._wait_for_registration(mock_cfn_client, REGISTRATION_TOKEN, True)

    mock_cfn_client.describe_type_registration.assert_called_once_with(
        RegistrationToken=REGISTRATION_TOKEN
    )
    mock_cfn_client.set_type_default_version.assert_not_called()
    mock_waiter.wait.assert_called_once_with(RegistrationToken=REGISTRATION_TOKEN) 

def _wait_for_stack(self, stack_id, waiter_name, success_msg):
        waiter = self.cfn_client.get_waiter(waiter_name)
        LOG.debug("Waiting for stack '%s'", stack_id)
        try:
            waiter.wait(
                StackName=stack_id, WaiterConfig={"Delay": 5, "MaxAttempts": 200}
            )
        except WaiterError as e:
            LOG.debug("Waiter failed for stack '%s'", stack_id, exc_info=e)
            LOG.critical(
                "Failed to create or update the '%s' stack. "
                "This stack is in your account, so you may be able to self-help by "
                "looking at '%s'. Otherwise, please reach out to CloudFormation.",
                stack_id,
                stack_id,
            )
            raise UploadError(
                "Failed to create or update the '{}' stack".format(stack_id)
            ) from e

        LOG.info(success_msg) 

def retry(function, *args, **kwargs):
    """ Retries a function up to max_retries, waiting `timeout` seconds between tries.
        This function is designed to retry both boto3 and fabric calls.  In the
        case of boto3, it is necessary because sometimes aws calls return too
        early and a resource needed by the next call is not yet available. """
    max_retries = kwargs.pop("max_retries", 10)
    timeout = kwargs.pop("timeout", 3)
    for i in range(max_retries):
        print (".", sys.stdout.flush())
        try:
            return function(*args, **kwargs)
        except (ClientError, NetworkError, WaiterError) as e:
            logger.debug("retrying %s, (~%s seconds elapsed)" % (function, i * 3))
            sleep(timeout)
    logger.error("hit max retries on %s" % function)
    raise e

#####################################################################################################################
#################################################### MAIN ###########################################################
##################################################################################################################### 

def retry(function, *args, **kwargs):
    """ Retries a function up to max_retries, waiting `timeout` seconds between tries.
        This function is designed to retry both boto3 and fabric calls.  In the
        case of boto3, it is necessary because sometimes aws calls return too
        early and a resource needed by the next call is not yet available. """
    max_retries = kwargs.pop("max_retries", 20)
    timeout = kwargs.pop("timeout", 0.25)
    for attempt in range(max_retries):
        try:
            ret = yield thread_pool.submit(function, *args, **kwargs)
            return ret
        except (ClientError, WaiterError) as e:
            app_log.warn("encountered %s, waiting for %s seconds before retrying..." % (type(e), timeout) )
            yield sleep(timeout)
    else:
         print("Failure in %s with args %s and kwargs %s" % (function.__name__, args, kwargs))
         #raise e 

def create_container(
        self, container_name: str, acl: str = None, meta_data: MetaData = None
    ) -> Container:
        if meta_data:
            logger.info(messages.OPTION_NOT_SUPPORTED, "meta_data")

        # Required parameters
        params = {
            "Bucket": container_name,
        }  # type: Dict[Any, Any]

        if acl:
            params["ACL"] = acl.lower()

        # TODO: BUG: Creating S3 bucket in us-east-1
        # See https://github.com/boto/boto3/issues/125
        if self.region != "us-east-1":
            params["CreateBucketConfiguration"] = {
                "LocationConstraint": self.region,
            }

        logger.debug("params=%s", params)

        try:
            bucket = self.s3.create_bucket(**params)
        except ParamValidationError as err:
            msg = err.kwargs.get("report", messages.CONTAINER_NAME_INVALID)
            raise CloudStorageError(msg)

        try:
            bucket.wait_until_exists()
        except WaiterError as err:
            logger.error(err)

        return self._make_container(bucket) 

def key_exists(self, bucket_name, key_name, min_successes=3):
        try:
            self.wait_until_key_exists(
                    bucket_name, key_name, min_successes=min_successes)
            return True
        except (ClientError, WaiterError):
            return False 

def _get_bucket(self, bucket_name: str, validate: bool = True):
        """Get a S3 bucket.

        :param bucket_name: The Bucket's name identifier.
        :type bucket_name: str

        :param validate: If True, verify that the bucket exists.
        :type validate: bool

        :return: S3 bucket resource object.
        :rtype: :class:`boto3.s3.Bucket`

        :raises NotFoundError: If the bucket does not exist.
        :raises CloudStorageError: Boto 3 client error.
        """
        bucket = self.s3.Bucket(bucket_name)

        if validate:
            try:
                response = self.s3.meta.client.head_bucket(Bucket=bucket_name)
                logger.debug("response=%s", response)
            except ClientError as err:
                error_code = int(err.response["Error"]["Code"])
                if error_code == 404:
                    raise NotFoundError(messages.CONTAINER_NOT_FOUND % bucket_name)

                raise CloudStorageError(
                    "%s: %s"
                    % (err.response["Error"]["Code"], err.response["Error"]["Message"])
                )

            try:
                bucket.wait_until_exists()
            except WaiterError as err:
                logger.error(err)

        return bucket 

def _create_change_set(self):
        """Creates a Cloudformation change set from a template
        """
        LOGGER.debug("%s - calling _create_change_set for %s", self.account_id, self.stack_name)
        try:
            self.template_url = self.template_url if self.template_url is not None else self.get_template_url()
            if self.template_url:
                self.validate_template()
                self.client.create_change_set(
                    StackName=self.stack_name,
                    TemplateURL=self.template_url,
                    Parameters=self.parameters if self.parameters is not None else self.get_parameters(),
                    Capabilities=[
                        'CAPABILITY_NAMED_IAM',
                    ],
                    Tags=[
                        {
                            'Key': 'createdBy',
                            'Value': 'ADF'
                        }
                    ],
                    ChangeSetName=self.stack_name,
                    ChangeSetType=self._get_change_set_type())
                self._wait_change_set()
                return True
            return False
        except ClientError as error:
            raise GenericAccountConfigureError(error)
        except WaiterError as error:
            err = error.last_response
            if CloudFormation._change_set_failed_due_to_empty(err["Status"], err["StatusReason"]):
                LOGGER.debug("%s - The submitted information does not contain changes.", self.account_id)
                self._delete_change_set()
                return False

            LOGGER.error("%s - ERROR: %s", self.account_id, err["StatusReason"], exc_info=1)
            self._delete_change_set()
            raise 

def update_stack(self, service_name, cloudformation_template, changeset_id, cf_stack):
        try:
            cs_response = self.cloudformation_client.create_change_set(
                StackName=cf_stack.stack_name,
                TemplateBody=cloudformation_template,
                Capabilities=["CAPABILITY_IAM"],
                ChangeSetName=changeset_id
            )
            cs_id = cs_response['Id']
            changes = self.cloudformation_client.describe_change_set(StackName=cf_stack.stack_name, ChangeSetName=cs_id)

            # delay for 2 seconds while waiting for changeset to create
            time.sleep(2)

            print(colored("The following stack update changes to be applied:", 'cyan'))
            print(colored("================================================================================", 'cyan'))
            print(colored(changes, 'yellow'))
            print(colored("================================================================================", 'cyan'))
            perform_update = query_yes_no("Perform changes?")
            if perform_update:
                self.cloudformation_client.execute_change_set(
                    ChangeSetName=changeset_id,
                    StackName=cf_stack.stack_name
                )

                waiter = CloudformationWaiter(self.cloudformation)
                print(colored('Cloudformation stack update in progress. Please check the AWS console!', color='green'))
                print(colored('Waiting on stack update...', color='magenta'))
                waiter.waiter.wait(StackName=service_name)
                print(colored('Stack update finished!', color='green'))
            else:
                self.cloudformation_client.delete_change_set(StackName=cf_stack.stack_name, ChangeSetName=cs_id)
        except ClientError as e:
            raise NovaError(str(e))
        except WaiterError as e:
            raise NovaError(str(e)) 

def is_rate_limit_error(exception):
    if exception in [exceptions.ClientError, exceptions.WaiterError]:
        if isinstance(exception, exceptions.ClientError):
            error_code = exception.response['Error']['Code']
        elif isinstance(exception, exceptions.WaiterError):
            error_code = exception.last_response['Error']['Code']
        if error_code in ['Throttling', 'RequestLimitExceeded']:
            print('AWS rate-limit encountered!')
            return True
    return False 

def retry_on_rate_limiting(e: Exception):
    """ Returns 'True' if a rate limiting error occurs and raises the exception otherwise
    """
    if isinstance(e, ClientError):
        error_code = e.response['Error']['Code']
    elif isinstance(e, WaiterError):
        error_code = e.last_response['Error']['Code']
    else:
        raise e
    if error_code in ['Throttling', 'RequestLimitExceeded']:
        log.warning('AWS API Limiting error: {}'.format(error_code))
        return True
    raise e 

def retry_on_rate_limiting(e: Exception):
    """ Returns 'True' if a rate limiting error occurs and raises the exception otherwise
    """
    if isinstance(e, ClientError):
        error_code = e.response['Error']['Code']
    elif isinstance(e, WaiterError):
        error_code = e.last_response['Error']['Code']
    else:
        raise e
    if error_code in ['Throttling', 'RequestLimitExceeded']:
        log.warning('AWS API Limiting error: {}'.format(error_code))
        return True
    raise e 

def test__wait_for_stack_failure(uploader):
    e = WaiterError(
        name="StackCreateComplete",
        reason="Waiter encountered a terminal failure state",
        last_response=None,
    )
    mock_waiter = uploader.cfn_client.get_waiter.return_value
    mock_waiter.wait.side_effect = e

    with pytest.raises(UploadError) as excinfo:
        uploader._wait_for_stack("stack-foo", "StackCreateComplete", "success-msg")

    mock_waiter.wait.assert_called_once_with(StackName="stack-foo", WaiterConfig=ANY)
    assert excinfo.value.__cause__ is e 

def retry_on_rate_limiting(e: Exception):
    """ Returns 'True' if a rate limiting error occurs and raises the exception otherwise
    """
    if isinstance(e, ClientError):
        error_code = e.response['Error']['Code']
    elif isinstance(e, WaiterError):
        error_code = e.last_response['Error']['Code']
    else:
        raise e
    if error_code in ['Throttling', 'RequestLimitExceeded']:
        log.warning('AWS API Limiting error: {}'.format(error_code))
        return True
    raise e 

def test_prints_error_message_but_exits_successfully_for_no_changes(capsys, logger, mocker):
    cf_client_mock = Mock()
    change_set = ChangeSet(STACK, cf_client_mock)
    status_reason = "The submitted information didn't contain changes. " \
                    "Submit different information to create a change set."

    error = WaiterError('name', 'reason', {'StatusReason': status_reason})
    cf_client_mock.get_waiter.return_value.wait.side_effect = error

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)
    logger.info.assert_called_with(status_reason) 

def test_prints_error_message_and_does_not_fail_without_StatusReason(capsys, logger):
    cf_client_mock = Mock()
    change_set = ChangeSet(STACK, cf_client_mock)

    error = WaiterError('name', 'reason', {})
    cf_client_mock.get_waiter.return_value.wait.side_effect = error

    with pytest.raises(SystemExit) as pytest_wrapped_e:
        change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)
    logger.info.assert_called()
    assert pytest_wrapped_e.value.code == 1 

def test_prints_error_message_for_failed_submit_and_exits(capsys, logger):
    cf_client_mock = Mock()
    change_set = ChangeSet(STACK, cf_client_mock)

    error = WaiterError('name', 'reason', {'StatusReason': 'StatusReason'})
    cf_client_mock.get_waiter.return_value.wait.side_effect = error

    with pytest.raises(SystemExit) as pytest_wrapped_e:
        change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)
    logger.info.assert_called_with('StatusReason')
    assert pytest_wrapped_e.value.code == 1 

def create_or_update_stack(self, managed_policy_arns: list, output: AbstractOutputWriter):
        """Creates or updates an instance profile.
        It was moved to a separate stack because creating of an instance profile resource takes 2 minutes.
        """
        # check that policies exist
        iam = boto3.client('iam', region_name=self._region)
        for policy_arn in managed_policy_arns:
            # if the policy doesn't exist, an error will be raised
            iam.get_policy(PolicyArn=policy_arn)

        template = prepare_instance_profile_template(managed_policy_arns)

        stack = Stack.get_by_name(self._cf, self._stack_name)
        try:
            if stack:
                # update the stack and wait until it will be updated
                self._update_stack(template, output)
            else:
                # create the stack and wait until it will be created
                self._create_stack(template, output)

            stack = Stack.get_by_name(self._cf, self._stack_name)
        except WaiterError:
            stack = None

        if not stack or stack.status not in ['CREATE_COMPLETE', 'UPDATE_COMPLETE']:
            raise ValueError('Stack "%s" was not created.\n'
                             'Please, see CloudFormation logs for the details.' % self._stack_name)

        profile_arn = [row['OutputValue'] for row in stack.outputs if row['OutputKey'] == 'ProfileArn'][0]

        return profile_arn 

def validate_and_find_master_dns(session, parsed_globals, cluster_id):
    """
    Utility method for ssh, socks, put and get command.
    Check if the cluster to be connected to is
     terminated or being terminated.
    Check if the cluster is running.
    Find master instance public dns of a given cluster.
    Return the latest created master instance public dns name.
    Throw MasterDNSNotAvailableError or ClusterTerminatedError.
    """
    cluster_state = emrutils.get_cluster_state(
        session, parsed_globals, cluster_id)

    if cluster_state in constants.TERMINATED_STATES:
        raise exceptions.ClusterTerminatedError

    emr = emrutils.get_client(session, parsed_globals)

    try:
        cluster_running_waiter = emr.get_waiter('cluster_running')
        if cluster_state in constants.STARTING_STATES:
            print("Waiting for the cluster to start.")
        cluster_running_waiter.wait(ClusterId=cluster_id)
    except WaiterError:
        raise exceptions.MasterDNSNotAvailableError

    return emrutils.find_master_dns(
        session=session, cluster_id=cluster_id,
        parsed_globals=parsed_globals) 

def key_not_exists(self, bucket_name, key_name, min_successes=3):
        try:
            self.wait_until_key_not_exists(
                    bucket_name, key_name, min_successes=min_successes)
            return True
        except (ClientError, WaiterError):
            return False 

def _wait_for_registration(cfn_client, registration_token, set_default):
        registration_waiter = cfn_client.get_waiter("type_registration_complete")
        try:
            LOG.warning(
                "Successfully submitted type. "
                "Waiting for registration with token '%s' to complete.",
                registration_token,
            )
            registration_waiter.wait(RegistrationToken=registration_token)
        except WaiterError as e:
            LOG.warning(
                "Failed to register the type with registration token '%s'.",
                registration_token,
            )
            try:
                response = cfn_client.describe_type_registration(
                    RegistrationToken=registration_token
                )
            except ClientError as describe_error:
                LOG.debug(
                    "Describing type registration resulted in unknown ClientError",
                    exc_info=e,
                )
                raise DownstreamError(
                    "Error describing type registration"
                ) from describe_error
            LOG.warning(
                "Please see response for additional information: '%s'", response
            )
            raise DownstreamError("Type registration error") from e
        LOG.warning("Registration complete.")
        response = cfn_client.describe_type_registration(
            RegistrationToken=registration_token
        )
        LOG.warning(response)
        if set_default:
            arn = response["TypeVersionArn"]
            try:
                cfn_client.set_type_default_version(Arn=arn)
            except ClientError as e:
                LOG.debug(
                    "Setting default version resulted in unknown ClientError",
                    exc_info=e,
                )
                raise DownstreamError("Error setting default version") from e
            LOG.warning("Set default version to '%s", arn) 

def purge_prefix(cloudfront, distribution: dict, origin: dict, prefix: str) -> None:
    distribution_id     = distribution["Id"]
    distribution_domain = domain_names(distribution)[0]

    # Purge everything starting with the prefix, after removing any implicit
    # origin path from the prefix.  If there is no prefix (e.g. the empty
    # string), we'll purge everything in the distribution.  Top-level keys
    # require a leading slash for proper invalidation.
    purge_prefix = "/%s*" % remove_origin_path(origin, prefix)

    print("Purging %s from CloudFront distribution %s (%s)… " % (purge_prefix, distribution_domain, distribution_id),
        end = "", flush = True)

    # Send the invalidation request.
    #
    # This is purposely a single path invalidation due to how AWS charges:
    #    https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html
    invalidation = cloudfront.create_invalidation(
        DistributionId    = distribution_id,
        InvalidationBatch = {
            "Paths": {
                "Quantity": 1,
                "Items": [ purge_prefix ]
            },
            "CallerReference": str(time())
        })

    # Wait up to 2 minutes for the invalidation to complete so we know it happened.
    invalidation_id = invalidation["Invalidation"]["Id"]
    waiter_config = {
        "Delay": 5,         # seconds
        "MaxAttempts": 12 * 2,
    }

    start = time()

    try:
        cloudfront.get_waiter('invalidation_completed').wait(
            Id             = invalidation_id,
            DistributionId = distribution_id,
            WaiterConfig   = waiter_config)
    except WaiterError as e:
        print("not yet complete")
        warn("Warning: Invalidation %s did not complete within %ds, but it will probably do so soon."
            % (invalidation_id, waiter_config["Delay"] * waiter_config["MaxAttempts"]))
    else:
        print("done (in %.0fs)" % (time() - start)) 

def wait_for_log(self, log, log_bucket, log_key, waiter_delay=30, waiter_max_attempts=20):
        '''Wait for gzipped EMR logs to appear on S3. Note that EMR syncs logs to S3 every 5
        minutes, so this may take a long time.

        Args:
            log_bucket (str): S3 bucket where log is expected to appear
            log_key (str): S3 key for the log file
            waiter_delay (int): How long to wait between attempts to check S3 for the log file
            waiter_max_attempts (int): Number of attempts before giving up on waiting

        Raises:
            EmrError: Raised if we waited the full duration and the logs did not appear

        Returns:
            str: contents of the log file
        '''
        check.str_param(log_bucket, 'log_bucket')
        check.str_param(log_key, 'log_key')
        check.int_param(waiter_delay, 'waiter_delay')
        check.int_param(waiter_max_attempts, 'waiter_max_attempts')

        log.info(
            'Attempting to get log: s3://{log_bucket}/{log_key}'.format(
                log_bucket=log_bucket, log_key=log_key
            )
        )

        s3 = _wrap_aws_client(boto3.client('s3'), min_backoff=self.check_cluster_every)
        waiter = s3.get_waiter('object_exists')
        try:
            waiter.wait(
                Bucket=log_bucket,
                Key=log_key,
                WaiterConfig={'Delay': waiter_delay, 'MaxAttempts': waiter_max_attempts},
            )
        except WaiterError as err:
            six.raise_from(
                EmrError('EMR log file did not appear on S3 after waiting'), err,
            )
        obj = BytesIO(s3.get_object(Bucket=log_bucket, Key=log_key)['Body'].read())
        gzip_file = gzip.GzipFile(fileobj=obj)
        return gzip_file.read().decode('utf-8') 

def create(self):
        try:
            self._aws_manager.create_and_upload_stack_template(self._s3_bucket, self._service_manager.service, self._service_manager.environment)

            self._aws_manager.create_stack(
                self._service_manager.service_name,
                self.cloudformation_template
            )

            stack = self._aws_manager.get_stack(self._service_manager.service_name)

            if stack.stack_status == "CREATE_COMPLETE":
                if stack.outputs is not None:
                    for output in stack.outputs:
                        if output.get('OutputKey') == 'CodeDeployApp':
                            code_deploy_app_id = output.get('OutputValue')
                            print(colored("Found code-deploy app: %s" % code_deploy_app_id, color='green'))
                            self._service_manager.service.set_code_deploy_app(self._environment_name, code_deploy_app_id)
                            print(colored("Please update nova.yml environment with 'deployment_application_id' manually.", color='yellow'))
                            break

                        for stack in self._service_manager.environment.stacks:
                            if output.get('OutputKey') == stack.name.title() + 'DeploymentGroup':
                                code_deploy_app_id = output.get('OutputValue')
                                print(colored("Found code-deploy deployment group for stack '%s': %s" % (stack.name, code_deploy_app_id), color='green'))
                                self._service_manager.service.set_code_deploy_app(self._environment_name, code_deploy_app_id)
                                print(colored("Please update nova.yml environment with 'deployment_application_id' manually.", color='yellow'))
                            else:
                                print(colored("Unable to find code-deploy deployment group for stack '%s' in the stack output" % stack.name, 'yellow'))
                                print(colored("Please update nova.yml stack with the 'deployment_group' manually.", color='yellow'))

                    else:
                        print(colored('Unable to find code-deploy application in the stack output', 'yellow'))
                        print(colored("Please update nova.yml environment with 'deployment_application_id'", color='yellow'))
                        print(colored("and your stacks with their respective 'deployment_group' manually.", color='yellow'))
                else:
                    print(colored("Please update nova.yml environment with 'deployment_application_id'", color='yellow'))
                    print(colored("and your stacks with their respective 'deployment_group' manually.", color='yellow'))

            else:
                raise NovaError("Stack creation was un-successful: %s" % stack.stack_status_reason)
        except ClientError as e:
            raise NovaError(str(e))
        except WaiterError as e:
            raise NovaError(str(e))