Python botocore.client() Examples

def patch():
    """
    Patch botocore client so it generates subsegments
    when calling AWS services.
    """
    if hasattr(botocore.client, '_xray_enabled'):
        return
    setattr(botocore.client, '_xray_enabled', True)

    wrapt.wrap_function_wrapper(
        'botocore.client',
        'BaseClient._make_api_call',
        _xray_traced_botocore,
    )

    wrapt.wrap_function_wrapper(
        'botocore.endpoint',
        'Endpoint.prepare_request',
        inject_header,
    ) 

def client(self):
        """
        Returns a botocore dynamodb client
        """
        # botocore has a known issue where it will cache empty credentials
        # https://github.com/boto/botocore/blob/4d55c9b4142/botocore/credentials.py#L1016-L1021
        # if the client does not have credentials, we create a new client
        # otherwise the client is permanently poisoned in the case of metadata service flakiness when using IAM roles
        if not self._client or (self._client._request_signer and not self._client._request_signer._credentials):
            config = botocore.client.Config(
                parameter_validation=False,  # Disable unnecessary validation for performance
                connect_timeout=self._connect_timeout_seconds,
                read_timeout=self._read_timeout_seconds,
                max_pool_connections=self._max_pool_connections)
            self._client = self.session.create_client(SERVICE_NAME, self.region, endpoint_url=self.host, config=config)
        return self._client 

def fake_kms_client(keysize=32):
    mock_kms_client = MagicMock(__class__=botocore.client.BaseClient)
    mock_kms_client.generate_data_key.return_value = {
        "Plaintext": VALUES["data_keys"][keysize]["plaintext"],
        "CiphertextBlob": VALUES["data_keys"][keysize]["encrypted"],
        "KeyId": VALUES["arn"],
    }
    mock_kms_client.encrypt.return_value = {
        "CiphertextBlob": VALUES["data_keys"][keysize]["encrypted"],
        "KeyId": VALUES["arn"],
    }
    mock_kms_client.decrypt.return_value = {
        "Plaintext": VALUES["data_keys"][keysize]["plaintext"],
        "KeyId": VALUES["arn"],
    }
    return mock_kms_client 

def remove_route_tables(tag_name, ssn=False):
    try:
        client = boto3.client('ec2')
        rtables = client.describe_route_tables(Filters=[{'Name': 'tag-key', 'Values': [tag_name]}]).get('RouteTables')
        for rtable in rtables:
            if rtable:
                rtable_associations = rtable.get('Associations')
                rtable = rtable.get('RouteTableId')
                if ssn:
                    for association in rtable_associations:
                        client.disassociate_route_table(AssociationId=association.get('RouteTableAssociationId'))
                        print("Association {} has been removed".format(association.get('RouteTableAssociationId')))
                client.delete_route_table(RouteTableId=rtable)
                print("Route table {} has been removed".format(rtable))
            else:
                print("There are no route tables to remove")
    except Exception as err:
        logging.info("Unable to remove route table: " + str(err) + "\n Traceback: " + traceback.print_exc(
            file=sys.stdout))
        append_result(str({"error": "Unable to remove route table",
                           "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
        traceback.print_exc(file=sys.stdout) 

def stop_ec2(tag_name, tag_value):
    try:
        ec2 = boto3.resource('ec2')
        client = boto3.client('ec2')
        inst = ec2.instances.filter(
            Filters=[{'Name': 'instance-state-name', 'Values': ['running', 'pending']},
                     {'Name': 'tag:{}'.format(tag_name), 'Values': ['{}'.format(tag_value)]}])
        instances = list(inst)
        if instances:
            id_instances = list()
            for instance in instances:
                id_instances.append(instance.id)
            client.stop_instances(InstanceIds=id_instances)
            waiter = client.get_waiter('instance_stopped')
            waiter.wait(InstanceIds=id_instances)
            print("The instances {} have been stopped successfully".format(id_instances))
        else:
            print("There are no instances with {} name to stop".format(tag_value))
    except Exception as err:
        logging.info("Unable to stop EC2: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
        append_result(str({"error": "Unable to stop EC2",
                           "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
        traceback.print_exc(file=sys.stdout) 

def deregister_image(image_name='*'):
    try:
        resource = boto3.resource('ec2')
        client = boto3.client('ec2')
        for image in resource.images.filter(
                Filters=[{'Name': 'tag-value', 'Values': [os.environ['conf_service_base_name']]},
                         {'Name': 'tag-value', 'Values': [image_name]}]):
            client.deregister_image(ImageId=image.id)
            for device in image.block_device_mappings:
                if device.get('Ebs'):
                    client.delete_snapshot(SnapshotId=device.get('Ebs').get('SnapshotId'))
            print("Notebook AMI {} has been deregistered successfully".format(image.id))
    except Exception as err:
        logging.info("Unable to de-register image: " + str(err) + "\n Traceback: " + traceback.print_exc(
            file=sys.stdout))
        append_result(str({"error": "Unable to de-register image",
                           "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
        traceback.print_exc(file=sys.stdout) 

def add_outbound_sg_rule(sg_id, rule):
    try:
        client = boto3.client('ec2')
        client.authorize_security_group_egress(
            GroupId=sg_id,
            IpPermissions=[rule]
        )
    except Exception as err:
        if err.response['Error']['Code'] == 'InvalidPermission.Duplicate':
            print("The following outbound rule is already exist:")
            print(str(rule))
        else:
            logging.info("Unable to add outbound rule to SG: " + str(err) + "\n Traceback: " + traceback.print_exc(
                file=sys.stdout))
            append_result(str({"error": "Unable to add outbound rule to SG",
                               "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
            traceback.print_exc(file=sys.stdout) 

def add_inbound_sg_rule(sg_id, rule):
    try:
        client = boto3.client('ec2')
        client.authorize_security_group_ingress(
            GroupId=sg_id,
            IpPermissions=[rule]
        )
    except Exception as err:
        if err.response['Error']['Code'] == 'InvalidPermission.Duplicate':
            print("The following inbound rule is already exist:")
            print(str(rule))
        else:
            logging.info("Unable to add inbound rule to SG: " + str(err) + "\n Traceback: " + traceback.print_exc(
                file=sys.stdout))
            append_result(str({"error": "Unable to add inbound rule to SG",
                               "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
            traceback.print_exc(file=sys.stdout) 

def remove_peering(tag_value):
    try:
        client = boto3.client('ec2')
        tag_name = os.environ['conf_service_base_name'].lower() + '-tag'
        if os.environ['conf_duo_vpc_enable'] == 'true':
            peering_id = client.describe_vpc_peering_connections(Filters=[
                {'Name': 'tag-key', 'Values': [tag_name]},
                {'Name': 'tag-value', 'Values': [tag_value]},
                {'Name': 'status-code', 'Values':
                    ['active']}]).get('VpcPeeringConnections')[0].get('VpcPeeringConnectionId')
            if peering_id:
                client.delete_vpc_peering_connection(VpcPeeringConnectionId=peering_id)
                print("Peering connection {} has been deleted successfully".format(peering_id))
            else:
                print("There are no peering connections to delete")
        else:
            print("There are no peering connections to delete because duo vpc option is disabled")
    except Exception as err:
        logging.info("Unable to remove peering connection: " + str(err) + "\n Traceback: " + traceback.print_exc(
            file=sys.stdout))
        append_result(str({"error": "Unable to remove peering connection",
                           "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
        traceback.print_exc(file=sys.stdout) 

def create_rt(vpc_id, infra_tag_name, infra_tag_value, secondary):
    try:
        tag = {"Key": infra_tag_name, "Value": infra_tag_value}
        route_table = []
        ec2 = boto3.client('ec2')
        rt = ec2.create_route_table(VpcId=vpc_id)
        rt_id = rt.get('RouteTable').get('RouteTableId')
        route_table.append(rt_id)
        print('Created Route-Table with ID: {}'.format(rt_id))
        create_tag(route_table, json.dumps(tag))
        if not secondary:
            ig = ec2.create_internet_gateway()
            ig_id = ig.get('InternetGateway').get('InternetGatewayId')
            route_table = []
            route_table.append(ig_id)
            create_tag(route_table, json.dumps(tag))
            ec2.attach_internet_gateway(InternetGatewayId=ig_id, VpcId=vpc_id)
            ec2.create_route(DestinationCidrBlock='0.0.0.0/0', RouteTableId=rt_id, GatewayId=ig_id)
        return rt_id
    except Exception as err:
        logging.info(
            "Unable to create Route Table: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
        append_result(str({"error": "Unable to create Route Table",
                           "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
        traceback.print_exc(file=sys.stdout) 

def get_mediapackage_metric_list(channel):
    print('[{}]: \t[{}], \t[{}]'.format('metric_name', 'primary', 'backup'))

    # Create CloudWatch client
    # cw_session = boto3.Session(profile_name = args['--profile'])
    cw_session = boto3.Session()
    cloudwatch = cw_session.client('cloudwatch')
    ## known metrics as of 8/19/2017
    all_metrics = {}
    all_metrics['EgressBytes'] = get_mediapackage_metric('EgressBytes', channel, cloudwatch)
    all_metrics['IngressBytes'] = get_mediapackage_metric('IngressBytes', channel, cloudwatch)
    all_metrics['EgressResponseTime'] = get_mediapackage_metric('EgressResponseTime', channel, cloudwatch)
    all_metrics['EgressRequestCount'] = get_mediapackage_metric('EgressRequestCount', channel, cloudwatch)
    all_metrics['IngressResponseTime'] = get_mediapackage_metric('IngressResponseTime', channel, cloudwatch)
    # print(all_metrics)
    return all_metrics 

def get_medialive_metric_list(channel):
    print('[{}]: \t[{}], \t[{}]'.format('metric_name', 'primary', 'backup'))

    # Create CloudWatch client
    # cw_session = boto3.Session(profile_name = args['--profile'])
    cw_session = boto3.Session()
    cloudwatch = cw_session.client('cloudwatch')
    ## known metrics as of 8/19/2017
    all_metrics = {}
    all_metrics['NetworkIn'] = get_medialive_metric('NetworkIn', channel, cloudwatch)
    all_metrics['NetworkOut'] = get_medialive_metric('NetworkOut', channel, cloudwatch)
    all_metrics['SvqTime'] = get_medialive_metric('SvqTime', channel, cloudwatch)
    all_metrics['FillMsec'] = get_medialive_metric('FillMsec', channel, cloudwatch)
    all_metrics['ActiveAlerts'] = get_medialive_metric('ActiveAlerts', channel, cloudwatch)
    all_metrics['DroppedFrames'] = get_medialive_metric('DroppedFrames', channel, cloudwatch)
    all_metrics['InputVideoFrameRate'] = get_medialive_metric('InputVideoFrameRate', channel, cloudwatch)
    # print(all_metrics)
    return all_metrics 

def start_ec2(tag_name, tag_value):
    try:
        ec2 = boto3.resource('ec2')
        client = boto3.client('ec2')
        inst = ec2.instances.filter(
            Filters=[{'Name': 'instance-state-name', 'Values': ['stopped']},
                     {'Name': 'tag:{}'.format(tag_name), 'Values': ['{}'.format(tag_value)]}])
        instances = list(inst)
        if instances:
            id_instances = list()
            for instance in instances:
                id_instances.append(instance.id)
            client.start_instances(InstanceIds=id_instances)
            waiter = client.get_waiter('instance_status_ok')
            waiter.wait(InstanceIds=id_instances)
            print("The instances {} have been started successfully".format(id_instances))
        else:
            print("There are no instances with {} name to start".format(tag_value))
    except Exception as err:
        logging.info("Unable to start EC2: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
        append_result(str({"error": "Unable to start EC2",
                           "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
        traceback.print_exc(file=sys.stdout) 

def s3_cleanup(bucket, cluster_name, user_name):
    s3_res = boto3.resource('s3', config=Config(signature_version='s3v4'))
    client = boto3.client('s3', config=Config(signature_version='s3v4'), region_name=os.environ['aws_region'])
    try:
        client.head_bucket(Bucket=bucket)
    except:
        print("There is no bucket {} or you do not permission to access it".format(bucket))
        sys.exit(0)
    try:
        resource = s3_res.Bucket(bucket)
        prefix = user_name + '/' + cluster_name + "/"
        for i in resource.objects.filter(Prefix=prefix):
            s3_res.Object(resource.name, i.key).delete()
    except Exception as err:
        logging.info("Unable to clean S3 bucket: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
        append_result(str({"error": "Unable to clean S3 bucket",
                           "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
        traceback.print_exc(file=sys.stdout) 

def set_default_client_config(self, client_config):
        """Sets the default config for creating clients

        :type client_config: botocore.client.Config
        :param client_config: The default client config object when creating
            clients. If the value is ``None`` then there is no default config
            object attached to the session.
        """
        self._client_config = client_config 

def get_waiter(self, waiterName: str) -> botocore.waiter.Waiter:
        """
        Get an AWS Batch service waiter

        :param waiterName: The name of the waiter.  The name should match
            the name (including the casing) of the key name in the waiter
            model file (typically this is CamelCasing).
        :type waiterName: str

        :return: a waiter object for the named AWS batch service
        :rtype: botocore.waiter.Waiter

        .. note::
            AWS batch might not have any waiters (until botocore PR-1307 is released).

            .. code-block:: python

                import boto3
                boto3.client('batch').waiter_names == []

        .. seealso::

            - https://boto3.amazonaws.com/v1/documentation/api/latest/guide/clients.html#waiters
            - https://github.com/boto/botocore/pull/1307
        """
        ... 

def client(self) -> Union[AwsBatchProtocol, botocore.client.BaseClient]:   # noqa: D402
        """
        An AWS API client for batch services, like ``boto3.client('batch')``

        :return: a boto3 'batch' client for the ``.region_name``
        :rtype: Union[AwsBatchProtocol, botocore.client.BaseClient]
        """
        return self.conn 

def get_default_client_config(self):
        """Retrieves the default config for creating clients

        :rtype: botocore.client.Config
        :returns: The default client config object when creating clients. If
            the value is ``None`` then there is no default config object
            attached to the session.
        """
        return self._client_config 

def get_default_client_config(self):
        """Retrieves the default config for creating clients

        :rtype: botocore.client.Config
        :returns: The default client config object when creating clients. If
            the value is ``None`` then there is no default config object
            attached to the session.
        """
        return self._client_config 

def set_default_client_config(self, client_config):
        """Sets the default config for creating clients

        :type client_config: botocore.client.Config
        :param client_config: The default client config object when creating
            clients. If the value is ``None`` then there is no default config
            object attached to the session.
        """
        self._client_config = client_config 

def get_default_client_config(self):
        """Retrieves the default config for creating clients

        :rtype: botocore.client.Config
        :returns: The default client config object when creating clients. If
            the value is ``None`` then there is no default config object
            attached to the session.
        """
        return self._client_config 

def _create_prepared_request(
        self,
        params: Dict,
        operation_model: Optional[Any],
    ) -> AWSPreparedRequest:
        request = create_request_object(params)
        self._sign_request(request)
        prepared_request = self.client._endpoint.prepare_request(request)
        if self._extra_headers is not None:
            prepared_request.headers.update(self._extra_headers)
        return prepared_request 

def logs_client(aws_region):
    with mock_logs():
        yield boto3.client("logs", region_name=aws_region) 

def iam_client(aws_region):
    with mock_iam():
        yield boto3.client("iam", region_name=aws_region) 

def ecs_client(aws_region):
    with mock_ecs():
        yield boto3.client("ecs", region_name=aws_region) 

def ec2_client(aws_region):
    with mock_ec2():
        yield boto3.client("ec2", region_name=aws_region) 

def batch_client(aws_region):
    with mock_batch():
        yield boto3.client("batch", region_name=aws_region) 

def get_cluster_python_version(region, bucket, user_name, cluster_name):
    s3_client = boto3.client('s3', config=Config(signature_version='s3v4'), region_name=region)
    s3_client.download_file(bucket, user_name + '/' + cluster_name + '/python_version', '/tmp/python_version') 

def yarn(args, yarn_dir):
    print("Downloading yarn configuration...")
    if args.region == 'cn-north-1':
        s3client = boto3.client('s3', config=Config(signature_version='s3v4'),
                                endpoint_url='https://s3.cn-north-1.amazonaws.com.cn', region_name=args.region)
        s3resource = boto3.resource('s3', config=Config(signature_version='s3v4'),
                                    endpoint_url='https://s3.cn-north-1.amazonaws.com.cn', region_name=args.region)
    else:
        s3client = boto3.client('s3', config=Config(signature_version='s3v4'), region_name=args.region)
        s3resource = boto3.resource('s3', config=Config(signature_version='s3v4'))
    get_files(s3client, s3resource, args.project_name + '/' + args.cluster_name + '/config/', args.bucket, yarn_dir)
    local('sudo mv ' + yarn_dir + args.project_name + '/' + args.cluster_name + '/config/* ' + yarn_dir)
    local('sudo rm -rf ' + yarn_dir + args.project_name + '/') 

def jars(args, emr_dir):
    print("Downloading jars...")
    s3_client = boto3.client('s3', config=Config(signature_version='s3v4'), region_name=args.region)
    s3_client.download_file(args.bucket, 'jars/' + args.emr_version + '/jars.tar.gz', '/tmp/jars.tar.gz')
    s3_client.download_file(args.bucket, 'jars/' + args.emr_version + '/jars-checksum.chk', '/tmp/jars-checksum.chk')
    if 'WARNING' in local('md5sum -c /tmp/jars-checksum.chk', capture=True):
        local('rm -f /tmp/jars.tar.gz')
        s3_client.download_file(args.bucket, 'jars/' + args.emr_version + '/jars.tar.gz', '/tmp/jars.tar.gz')
        if 'WARNING' in local('md5sum -c /tmp/jars-checksum.chk', capture=True):
            print("The checksum of jars.tar.gz is mismatched. It could be caused by aws network issue.")
            sys.exit(1)
    local('tar -zhxvf /tmp/jars.tar.gz -C ' + emr_dir)