Python botocore.client.ClientError() Examples

def discover_resources(**kwargs):

    discovered_buckets = []    
    for handler in AWSHandler.objects.all():
        set_progress('Connecting to Amazon S3 for handler: {}'.format(handler))
        conn = boto3.resource(
            's3',
            aws_access_key_id=handler.serviceaccount,
            aws_secret_access_key=handler.servicepasswd,
        )

        try:
            for bucket in conn.buckets.all():
                discovered_buckets.append({
                    "s3_bucket_name": bucket.name,
                    "aws_rh_id": handler.id,
                    "created_in_s3": str(bucket.creation_date)
                })
        except ClientError as e:
            set_progress('AWS ClientError: {}'.format(e))
            continue
            
    return discovered_buckets 

def _function_alias(name, version, alias=LIVE):
    try:
        logger.info('creating function alias {0} for {1}:{2}'.format(
            alias, name, version))
        arn = aws_lambda('create_alias',
                         FunctionName=name,
                         FunctionVersion=version,
                         Name=alias,
                         query='AliasArn')
    except ClientError:
        logger.info('alias {0} exists. updating {0} -> {1}:{2}'.format(
            alias, name, version))
        arn = aws_lambda('update_alias',
                         FunctionName=name,
                         FunctionVersion=version,
                         Name=alias,
                         query='AliasArn')
    return arn 

def role():
    new_role = False
    try:
        logger.info('finding role')
        iam('get_role', RoleName='cronyo')
    except ClientError:
        logger.info('role not found. creating')
        iam('create_role', RoleName='cronyo',
            AssumeRolePolicyDocument=ASSUMED_ROLE_POLICY)
        new_role = True

    role_arn = iam('get_role', RoleName='cronyo', query='Role.Arn')
    logger.debug('role_arn={}'.format(role_arn))

    logger.info('updating role policy')

    iam('put_role_policy', RoleName='cronyo', PolicyName='cronyo',
        PolicyDocument=POLICY)

    if new_role:
        from time import sleep
        logger.info('waiting for role policy propagation')
        sleep(5)

    return role_arn 

def getMediaConvertJob(id, JOBTABLE):
    """
    Retrieve a job data structure from dynamodb
    :param id:  id of the job
    :param JOBTABLE: Name of the dynamodb job table for this stack
    """    
    try:
        table = DYNAMO_CLIENT.Table(JOBTABLE)
        response = table.get_item(Key={'id': id}, ConsistentRead=True)
    
    except ClientError as e:
        print(e.response['Error']['Message'])
    else:
        if 'Item' not in response:
            return None
        else:
            item = response['Item']
            print("GetItem succeeded:")
            #print(json.dumps(item, indent=4, cls=DecimalEncoder))
            return item 

def read_metadata_from_s3(bucket, key):
    try:
        obj = s3_client.get_object(
            Bucket=bucket,
            Key=key
        )
    except ClientError as e:
        error = e.response['Error']['Message']
        logger.info("Exception occurred while reading asset metadata from s3: {e}".format(e=error))
        return {"Status": "Error", "Message": error}
    except Exception as e:
        logger.error("Exception occurred while reading asset metadata from s3")
        return {"Status": "Error", "Message": e}
    else:
        results = obj['Body'].read().decode('utf-8')
        return {"Status": "Success", "Object": results} 

def _function_alias(name, version, alias=LIVE):
    try:
        logger.info('creating function alias {0} for {1}:{2}'.format(
            alias, name, version))
        arn = aws_lambda('create_alias',
                         FunctionName=name,
                         FunctionVersion=version,
                         Name=alias,
                         query='AliasArn')
    except ClientError:
        logger.info('alias {0} exists. updating {0} -> {1}:{2}'.format(
            alias, name, version))
        arn = aws_lambda('update_alias',
                         FunctionName=name,
                         FunctionVersion=version,
                         Name=alias,
                         query='AliasArn')
    return arn 

def role():
    new_role = False
    try:
        logger.info('finding role')
        iam('get_role', RoleName='gimel')
    except ClientError:
        logger.info('role not found. creating')
        iam('create_role', RoleName='gimel',
            AssumeRolePolicyDocument=ASSUMED_ROLE_POLICY)
        new_role = True

    role_arn = iam('get_role', RoleName='gimel', query='Role.Arn')
    logger.debug('role_arn={}'.format(role_arn))

    logger.info('updating role policy')

    iam('put_role_policy', RoleName='gimel', PolicyName='gimel',
        PolicyDocument=POLICY)

    if new_role:
        from time import sleep
        logger.info('waiting for role policy propagation')
        sleep(5)

    return role_arn 

def test_make_api_call_throws_verbose_error_after_backoff(self, client_mock):
        response = AWSResponse(
            url='http://lyft.com',
            status_code=500,
            raw='',  # todo: use stream, like `botocore.tests.RawResponse`?
            headers={'x-amzn-RequestId': 'abcdef'},
        )
        response._content = json.dumps({'message': 'There is a problem', '__type': 'InternalServerError'}).encode('utf-8')
        client_mock._endpoint.http_session.send.return_value = response

        c = Connection()

        with self.assertRaises(ClientError):
            try:
                c._make_api_call('CreateTable', {'TableName': 'MyTable'})
            except Exception as e:
                self.assertEqual(
                    str(e),
                    'An error occurred (InternalServerError) on request (abcdef) on table (MyTable) when calling the CreateTable operation: There is a problem'
                )
                raise 

def test_describe_table(self):
        """
        Connection.describe_table
        """
        with patch(PATCH_METHOD) as req:
            req.return_value = DESCRIBE_TABLE_DATA
            conn = Connection(self.region)
            conn.describe_table(self.test_table_name)
            self.assertEqual(req.call_args[0][1], {'TableName': 'ci-table'})

        with self.assertRaises(TableDoesNotExist):
            with patch(PATCH_METHOD) as req:
                req.side_effect = ClientError({'Error': {'Code': 'ResourceNotFoundException', 'Message': 'Not Found'}}, "DescribeTable")
                conn = Connection(self.region)
                conn.describe_table(self.test_table_name)

        with self.assertRaises(TableDoesNotExist):
            with patch(PATCH_METHOD) as req:
                req.side_effect = ValueError()
                conn = Connection(self.region)
                conn.describe_table(self.test_table_name) 

def test_model_subclass_attributes_inherited_on_create(self):
        scope_args = {'count': 0}

        def fake_dynamodb(*args, **kwargs):
            if scope_args['count'] == 0:
                scope_args['count'] += 1
                raise ClientError({'Error': {'Code': 'ResourceNotFoundException', 'Message': 'Not Found'}},
                                  "DescribeTable")
            return {}

        fake_db = MagicMock()
        fake_db.side_effect = fake_dynamodb

        with patch(PATCH_METHOD, new=fake_db) as req:
            Dog.create_table(read_capacity_units=2, write_capacity_units=2)

            actual = req.call_args_list[1][0][1]

            self.assertEqual(actual['TableName'], DOG_TABLE_DATA['Table']['TableName'])
            self.assert_dict_lists_equal(actual['KeySchema'], DOG_TABLE_DATA['Table']['KeySchema'])
            self.assert_dict_lists_equal(actual['AttributeDefinitions'],
                                         DOG_TABLE_DATA['Table']['AttributeDefinitions']) 

def delete_s3_objects(keys):
    objects = []
    for key in keys:
        objects.append({"Key": key})
    try:
        response = s3_client.delete_objects(
            Bucket=dataplane_s3_bucket,
            Delete={
                'Objects': objects
            }
        )
    except ClientError as e:
        error = e.response['Error']['Message']
        logger.info("Exception occurred while deleting asset metadata from s3: {e}".format(e=error))
        return {"Status": "Error", "Message": error}
    except Exception as e:
        logger.error("Exception occurred while deleting asset metadata from s3")
        return {"Status": "Error", "Message": e}
    else:
        return {"Status": "Success", "Message": response} 

def s3_verify_bucket_exists(self, bucketname):
        """
        Verifies whether a bucket with the given bucketname exists
        Args:
            bucketname : The bucket name to be verified

        Returns:
              bool: True if bucket exists, False otherwise

        """
        try:
            self.s3_resource.meta.client.head_bucket(Bucket=bucketname)
            logger.info(f"{bucketname} exists")
            return True
        except ClientError:
            logger.info(f"{bucketname} does not exist")
            return False 

def upload():
    """
    Generate a pre-signed URL that can be used to upload media files to S3 from a web application

    Returns:
        Pre-signed S3 URL for uploading files to S3 from a web application
    Raises:
        ChaliceViewError - 500
    """
    print('/upload request: '+app.current_request.raw_body.decode())
    region = os.environ['AWS_REGION']
    s3 = boto3.client('s3', region_name=region, config = Config(signature_version = 's3v4', s3={'addressing_style': 'virtual'}))
    # limit uploads to 5GB
    max_upload_size = 5368709120
    try:
        response = s3.generate_presigned_post(
            Bucket=(json.loads(app.current_request.raw_body.decode())['S3Bucket']),
            Key=(json.loads(app.current_request.raw_body.decode())['S3Key']),
            Conditions=[["content-length-range", 0, max_upload_size ]],
            ExpiresIn=3600
        )
    except ClientError as e:
        logging.info(e)
        raise ChaliceViewError(
            "Unable to generate pre-signed S3 URL for uploading media: {error}".format(error=e))
    except Exception as e:
        logging.info(e)
        raise ChaliceViewError(
            "Unable to generate pre-signed S3 URL for uploading media: {error}".format(error=e))
    else:
        print("presigned url generated: ", response)
        return response

# TODO: Change the name of this method - "download" is too vague 

def write_metadata_to_s3(bucket, key, data):
    encoded = json.dumps(data, cls=DecimalEncoder)
    try:
        s3_client.put_object(Bucket=bucket, Key=key, Body=encoded)
    except ClientError as e:
        error = e.response['Error']['Message']
        logger.info("Exception occurred while writing asset metadata to s3: {e}".format(e=error))
        return {"Status": "Error", "Message": error}
    except Exception as e:
        logger.error("Exception occurred while writing asset metadata to s3")
        return {"Status": "Error", "Message": e}
    else:
        logger.info("Wrote asset metadata to s3")
        return {"Status": "Success"} 

def process_config(self, client, config):
        try:
            client.delete_launch_configuration(
                LaunchConfigurationName=config[
                    'LaunchConfigurationName'])
        except ClientError as e:
            # Catch already deleted
            if e.response['Error']['Code'] == 'ValidationError':
                return
            raise 

def process_asg(self, client, asg):
        force_delete = self.data.get('force', False)
        try:
            self.manager.retry(
                client.delete_auto_scaling_group,
                AutoScalingGroupName=asg['AutoScalingGroupName'],
                ForceDelete=force_delete)
        except ClientError as e:
            if e.response['Error']['Code'] == 'ValidationError':
                return
            raise 

def download():
    """
    Generate a pre-signed URL that can be used to download media files from S3.

    Returns:
        Pre-signed S3 URL for downloading files from S3 to a web application.
    Raises:
        ChaliceViewError - 500
    """
    print('/download request: '+app.current_request.raw_body.decode())
    s3 = boto3.client('s3')
    # expire the URL in
    try:
        response = s3.generate_presigned_url('get_object',
                                             Params={'Bucket': json.loads(app.current_request.raw_body.decode())['S3Bucket'],
                                                     'Key': json.loads(app.current_request.raw_body.decode())['S3Key']},
                                             ExpiresIn=3600)
    except ClientError as e:
        logging.info(e)
        raise ChaliceViewError(
            "Unable to generate pre-signed S3 URL for downloading media: {error}".format(error=e))
    except Exception as e:
        logging.info(e)
        raise ChaliceViewError(
            "Unable to generate pre-signed S3 URL for downloading media: {error}".format(error=e))
    else:
        return response

# TODO: Change the name of this method 

def put_dynamo_main(dynamo_object):
    print("put_dynamo to table: " + str(DYNAMO_MAIN))
    table = dynamodb.Table(DYNAMO_MAIN)
    try:
        response = table.put_item(
            Item=dynamo_object,
            ConditionExpression='attribute_not_exists(id_filename)'
        )
        print("dynamo put_item succeeded: {}".format(response))
    except ClientError as e:
        # Ignore the ConditionalCheckFailedException, bubble up other exceptions.
        if e.response['Error']['Code'] != 'ConditionalCheckFailedException':
            raise 

def process_asg(self, asg):
        """Multistep process to stop an asg aprori of setup

        - suspend processes
        - stop instances
        """
        session = local_session(self.manager.session_factory)
        asg_client = session.client('autoscaling')
        processes = list(self.ASG_PROCESSES.difference(
            self.data.get('exclude', ())))

        try:
            self.manager.retry(
                asg_client.suspend_processes,
                ScalingProcesses=processes,
                AutoScalingGroupName=asg['AutoScalingGroupName'])
        except ClientError as e:
            if e.response['Error']['Code'] == 'ValidationError':
                return
            raise
        ec2_client = session.client('ec2')
        try:
            instance_ids = [i['InstanceId'] for i in asg['Instances']]
            if not instance_ids:
                return
            retry = get_retry((
                'RequestLimitExceeded', 'Client.RequestLimitExceeded'))
            retry(ec2_client.stop_instances, InstanceIds=instance_ids)
        except ClientError as e:
            if e.response['Error']['Code'] in (
                    'InvalidInstanceID.NotFound',
                    'IncorrectInstanceState'):
                self.log.warning("Erroring stopping asg instances %s %s" % (
                    asg['AutoScalingGroupName'], e))
                return
            raise 

def _get_target_arn(name):
    try:
        function_arn = aws_lambda('get_function',
                                  FunctionName=name,
                                  query='Configuration.FunctionArn')
    except ClientError:
        function_arn = None
    return function_arn 

def put_dynamo_list(dynamo_object, shot_object, scoreboard, image_filename, segment_datetime, trigger_type):
    print("put_dynamo to table: " + str(DYNAMO_LIST))
    table = dynamodb.Table(DYNAMO_LIST)
    this_uuid = str(uuid.uuid4())
    label_sort = str(int((datetime.datetime.utcnow() - datetime.datetime(1970,1,1)).total_seconds())) + this_uuid
    datetime_start = datetime.datetime.strptime(segment_datetime, '%Y-%m-%dT%H:%M:%S.%fZ')
    action_time = 15
    if trigger_type == 'score': 
        action_time = 25
    try:
        response = table.put_item(
            Item={
                    'entry_id': this_uuid,
                    'timestamp_created': int((datetime.datetime.utcnow() - datetime.datetime(1970,1,1)).total_seconds()),
                    'timestamp_ttl': int((datetime.datetime.utcnow() - datetime.datetime(1970,1,1)).total_seconds() + 900 ),
                    'label': 'scoreboard',
                    'end_time' : datetime_start.strftime("%Y-%m-%d %H:%M:%S").replace(' ', 'T') + '+00:00',
                    'start_time' : (datetime_start - datetime.timedelta(seconds=float(action_time))).strftime("%Y-%m-%d %H:%M:%S").replace(' ', 'T') + '+00:00',
                    'label_image': image_filename, 
                    'scoreboard': scoreboard, 
                    'shot_object': shot_object, 
                    'dynamo_object': dynamo_object, 
                    'trigger_type': trigger_type,
                    'label_sort': label_sort,
            },
            ConditionExpression='attribute_not_exists(entry_id)'
        )
        print("dynamo put_item succeeded: {}".format(response))
    except ClientError as e:
        # Ignore the ConditionalCheckFailedException, bubble up other exceptions.
        if e.response['Error']['Code'] != 'ConditionalCheckFailedException':
            raise 

def get_s3file(BUCKET_NAME, KEY, LOCALFILE):
    try:
        s3.Bucket(BUCKET_NAME).download_file(KEY, LOCALFILE)
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == "404":
            print("The object does not exist. 404")
        else:
            raise 

def _clear_method(api_id, resource_id, http_method):
    try:
        method = apigateway('get_method', restApiId=api_id,
                            resourceId=resource_id,
                            httpMethod=http_method)
    except ClientError:
        method = None
    if method:
        apigateway('delete_method', restApiId=api_id, resourceId=resource_id,
                   httpMethod=http_method) 

def assert_condition_check_fails():
    try:
        yield
    except (PutError, UpdateError, DeleteError) as e:
        assert isinstance(e.cause, ClientError)
        assert e.cause_response_code == "ConditionalCheckFailedException"
    except TransactWriteError as e:
        assert isinstance(e.cause, ClientError)
        assert e.cause_response_code == "TransactionCanceledException"
        assert "ConditionalCheckFailed" in e.cause_response_message
    else:
        raise AssertionError("The version attribute conditional check should have failed.") 

def _bucket_exists(self):
        """Check if the bucket exists."""
        try:
            self.s3client.get_bucket_location(Bucket=self.bucket)
            return True
        except ClientError as error:
            LOG.error(error)
            return False 

def swf_exception_wrapper():
    try:
        yield
    except ClientError as err:
        err_type = err.response['Error'].get('Code', 'SWFResponseError')
        err_msg = err.response['Error'].get(
            'Message', 'No error message provided...')

        raise _swf_fault_exception.get(err_type, SWFResponseError)(err_msg) 

def put(name,
        cron_expression,
        function_name,
        target_input={},
        description=None):

    logger.info("finding lambda function {}".format(function_name))
    target_arn = \
        _get_target_arn(function_name) or \
        _get_target_arn(_namespaced(function_name))
    if not target_arn:
        logger.error("unable to find lambda function for {}".format(function_name))
        return

    logger.debug(
        "create / update cron rule {0}: {1} for target {2}".format(
            name,
            cron_expression,
            target_arn
        )
    )
    if description:
        rule = events("put_rule",
                      Name=name,
                      ScheduleExpression=cron_expression,
                      Description=description)
    else:
        rule = events("put_rule",
                      Name=name,
                      ScheduleExpression=cron_expression)
    events(
        "put_targets",
        Rule=name,
        Targets=[
            {
                "Id": "1",
                "Arn": target_arn,
                "Input": json.dumps(target_input)
            }
        ]
    )
    try:
        logger.debug("setting lambda permission")
        source_arn = rule["RuleArn"]
        if source_arn.find(NAMESPACE) > 0:
            rule_prefix = rule["RuleArn"].split("/{}".format(NAMESPACE))[0]
            source_arn = "{}/{}*".format(rule_prefix, NAMESPACE)
        logger.debug("lambda permission SourceArn:{}".format(source_arn))
        aws_lambda(
            "add_permission",
            FunctionName=target_arn,
            Action="lambda:InvokeFunction",
            Principal="events.amazonaws.com",
            SourceArn=source_arn,
            StatementId=hashlib.sha1(source_arn.encode("utf-8")).hexdigest()
        )
    except ClientError as error:
        logger.debug("permission already set. {}".format(error))

    for rule in _find([name]):
        logger.info("rule created/updated:\n{}".format(yaml.dump(_export_rule(rule)))) 

def create_update_lambda(role_arn, wiring):
    name, handler, memory, timeout = (wiring[k] for k in ('FunctionName',
                                                          'Handler',
                                                          'MemorySize',
                                                          'Timeout'))
    try:
        logger.info('finding lambda function')
        function_arn = aws_lambda('get_function',
                                  FunctionName=name,
                                  query='Configuration.FunctionArn')
    except ClientError:
        function_arn = None
    if not function_arn:
        logger.info('creating new lambda function {}'.format(name))
        with open('cronyo.zip', 'rb') as zf:
            function_arn, version = aws_lambda('create_function',
                                               FunctionName=name,
                                               Runtime='python3.8',
                                               Role=role_arn,
                                               Handler=handler,
                                               MemorySize=memory,
                                               Timeout=timeout,
                                               Publish=True,
                                               Code={'ZipFile': zf.read()},
                                               query='[FunctionArn, Version]')
    else:
        logger.info('updating lambda function {}'.format(name))
        aws_lambda('update_function_configuration',
                   FunctionName=name,
                   Runtime='python3.8',
                   Role=role_arn,
                   Handler=handler,
                   MemorySize=memory,
                   Timeout=timeout)
        with open('cronyo.zip', 'rb') as zf:
            function_arn, version = aws_lambda('update_function_code',
                                               FunctionName=name,
                                               Publish=True,
                                               ZipFile=zf.read(),
                                               query='[FunctionArn, Version]')
    function_arn = _function_alias(name, version)
    _cleanup_old_versions(name)
    logger.debug('function_arn={} ; version={}'.format(function_arn, version))
    return function_arn 

def create_update_lambda(role_arn, wiring):
    name, handler, memory, timeout = (wiring[k] for k in ('FunctionName',
                                                          'Handler',
                                                          'MemorySize',
                                                          'Timeout'))
    try:
        logger.info('finding lambda function')
        function_arn = aws_lambda('get_function',
                                  FunctionName=name,
                                  query='Configuration.FunctionArn')
    except ClientError:
        function_arn = None
    if not function_arn:
        logger.info('creating new lambda function {}'.format(name))
        with open('gimel.zip', 'rb') as zf:
            function_arn, version = aws_lambda('create_function',
                                               FunctionName=name,
                                               Runtime='python3.8',
                                               Role=role_arn,
                                               Handler=handler,
                                               MemorySize=memory,
                                               Timeout=timeout,
                                               Publish=True,
                                               Code={'ZipFile': zf.read()},
                                               query='[FunctionArn, Version]')
    else:
        logger.info('updating lambda function {}'.format(name))
        aws_lambda('update_function_configuration',
                   FunctionName=name,
                   Runtime='python3.8',
                   Role=role_arn,
                   Handler=handler,
                   MemorySize=memory,
                   Timeout=timeout)
        with open('gimel.zip', 'rb') as zf:
            function_arn, version = aws_lambda('update_function_code',
                                               FunctionName=name,
                                               Publish=True,
                                               ZipFile=zf.read(),
                                               query='[FunctionArn, Version]')
    function_arn = _function_alias(name, version)
    _cleanup_old_versions(name)
    logger.debug('function_arn={} ; version={}'.format(function_arn, version))
    return function_arn 

def test_global_index(self):
        """
        Models.GlobalSecondaryIndex
        """
        self.assertIsNotNone(IndexedModel.email_index._hash_key_attribute())
        self.assertEqual(IndexedModel.email_index.Meta.projection.projection_type, AllProjection.projection_type)
        with patch(PATCH_METHOD) as req:
            req.return_value = INDEX_TABLE_DATA
            with self.assertRaises(ValueError):
                IndexedModel('foo', 'bar')
            IndexedModel._get_connection().describe_table()

        scope_args = {'count': 0}

        def fake_dynamodb(*args, **kwargs):
            if scope_args['count'] == 0:
                scope_args['count'] += 1
                raise ClientError({'Error': {'Code': 'ResourceNotFoundException', 'Message': 'Not Found'}},
                                  "DescribeTable")
            else:
                return {}

        fake_db = MagicMock()
        fake_db.side_effect = fake_dynamodb

        with patch(PATCH_METHOD, new=fake_db) as req:
            IndexedModel.create_table(read_capacity_units=2, write_capacity_units=2)
            params = {
                'AttributeDefinitions': [
                    {'attribute_name': 'email', 'attribute_type': 'S'},
                    {'attribute_name': 'numbers', 'attribute_type': 'NS'}
                ],
                'KeySchema': [
                    {'AttributeName': 'numbers', 'KeyType': 'RANGE'},
                    {'AttributeName': 'email', 'KeyType': 'HASH'}
                ]
            }
            schema = IndexedModel.email_index._get_schema()
            args = req.call_args[0][1]
            self.assertEqual(
                args['GlobalSecondaryIndexes'][0]['ProvisionedThroughput'],
                {
                    'ReadCapacityUnits': 2,
                    'WriteCapacityUnits': 1
                }
            )
            self.assert_dict_lists_equal(schema['key_schema'], params['KeySchema'])
            self.assert_dict_lists_equal(schema['attribute_definitions'], params['AttributeDefinitions'])