Python botocore.session.create_client() Examples

def test_map_parameter_grouping():
    """
    Test special parameters that have shape of map are recorded
    as a list of keys based on `para_whitelist.json`
    """
    ddb = session.create_client('dynamodb', region_name='us-west-2')
    response = {
        'ResponseMetadata': {
            'RequestId': REQUEST_ID,
            'HTTPStatusCode': 500,
        }
    }

    with Stubber(ddb) as stubber:
        stubber.add_response('batch_write_item', response, {'RequestItems': ANY})
        ddb.batch_write_item(RequestItems={'table1': [{}], 'table2': [{}]})

    subsegment = xray_recorder.current_segment().subsegments[0]
    assert subsegment.fault
    assert subsegment.http['response']['status'] == 500

    aws_meta = subsegment.aws
    assert sorted(aws_meta['table_names']) == ['table1', 'table2'] 

def _get_credentials():
    log.write("Refreshing credentials.")
    session = botocore.session.Session()
    credentials = session.get_credentials()
    return {
        "AWS_ACCESS_KEY_ID": credentials.access_key,
        "AWS_SECRET_ACCESS_KEY": credentials.secret_key,
        "AWS_SESSION_TOKEN": credentials.token,
        "AWS_DEFAULT_REGION": session.create_client("s3").meta.region_name
    } 

def _get_client(client_name, url, region, access, secret, ca_bundle):
    connection_data = {
        'config_file': (None, 'AWS_CONFIG_FILE', None, None),
        'region': ('region', 'AWS_DEFAULT_REGION', region, None),
    }
    session = botocore.session.get_session(connection_data)
    kwargs = {
        'region_name': region,
        'endpoint_url': url,
        'aws_access_key_id': access,
        'aws_secret_access_key': secret
    }
    if ca_bundle:
        try:
            kwargs['verify'] = types.Boolean()(ca_bundle)
        except Exception:
            kwargs['verify'] = ca_bundle
    return session.create_client(client_name, **kwargs) 

def test_only_dynamodb_calls_are_traced():
    """Test only a single subsegment is created for other AWS services.

    As the pynamodb patch applies the botocore patch as well, we need
    to ensure that only one subsegment is created for all calls not
    made by PynamoDB. As PynamoDB calls botocore differently than the
    botocore patch expects we also just get a single subsegment per
    PynamoDB call.
    """
    session = botocore.session.get_session()
    s3 = session.create_client('s3', region_name='us-west-2',
                               config=Config(signature_version=UNSIGNED))
    try:
        s3.get_bucket_location(Bucket='mybucket')
    except ClientError:
        pass

    subsegments = xray_recorder.current_segment().subsegments
    assert len(subsegments) == 1
    assert subsegments[0].name == 's3'
    assert len(subsegments[0].subsegments) == 0 

def test_sns_publish_parameters():
    sns = session.create_client('sns', region_name='us-west-2')
    response = {
        'ResponseMetadata': {
            'RequestId': REQUEST_ID,
            'HTTPStatusCode': 200,
        }
    }

    with Stubber(sns) as stubber:
        stubber.add_response('publish', response, {'TopicArn': 'myAmazingTopic', 'Message': 'myBodaciousMessage'})
        sns.publish(TopicArn='myAmazingTopic', Message='myBodaciousMessage')

    subsegment = xray_recorder.current_segment().subsegments[0]
    assert subsegment.http['response']['status'] == 200

    aws_meta = subsegment.aws
    assert aws_meta['topic_arn'] == 'myAmazingTopic'
    assert aws_meta['request_id'] == REQUEST_ID
    assert aws_meta['region'] == 'us-west-2'
    assert aws_meta['operation'] == 'Publish' 

def get_existing_graph(session: Optional[botocore.session.Session], account: Optional[str], debug=False) -> Graph:
    """Returns a Graph object stored on-disk in a standard location (per-OS, using the get_storage_root utility function
    in principalmapper.util.storage). Uses the session/account parameter to choose the directory from under the
    standard location.
    """
    if account is not None:
        dprint(debug, 'Loading account data based on parameter --account')
        graph = get_graph_from_disk(os.path.join(get_storage_root(), account))
    elif session is not None:
        dprint(debug, 'Loading account data using a botocore session object')
        stsclient = session.create_client('sts')
        response = stsclient.get_caller_identity()
        graph = get_graph_from_disk(os.path.join(get_storage_root(), response['Account']))
    else:
        raise ValueError('One of the parameters `account` or `session` must not be None')
    return graph 

def get_s3_content_and_delete(bucket, path, with_key=False):
    """ Get content from s3 key, and delete key afterwards.
    """
    if is_botocore():
        import botocore.session
        session = botocore.session.get_session()
        client = session.create_client('s3')
        key = client.get_object(Bucket=bucket, Key=path)
        content = key['Body'].read()
        client.delete_object(Bucket=bucket, Key=path)
    else:
        import boto
        # assuming boto=2.2.2
        bucket = boto.connect_s3().get_bucket(bucket, validate=False)
        key = bucket.get_key(path)
        content = key.get_contents_as_string()
        bucket.delete_key(path)
    return (content, key) if with_key else content 

def __init__(self, uri):
        self.is_botocore = is_botocore()
        if self.is_botocore:
            import botocore.session
            session = botocore.session.get_session()
            self.s3_client = session.create_client(
                's3',
                aws_access_key_id=self.AWS_ACCESS_KEY_ID,
                aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
                endpoint_url=self.AWS_ENDPOINT_URL,
                region_name=self.AWS_REGION_NAME,
                use_ssl=self.AWS_USE_SSL,
                verify=self.AWS_VERIFY
            )
        else:
            from boto.s3.connection import S3Connection
            self.S3Connection = S3Connection
        assert uri.startswith('s3://')
        self.bucket, self.prefix = uri[5:].split('/', 1) 

def get_s3_content_and_delete(bucket, path, with_key=False):
    """ Get content from s3 key, and delete key afterwards.
    """
    if is_botocore():
        import botocore.session
        session = botocore.session.get_session()
        client = session.create_client('s3')
        key = client.get_object(Bucket=bucket, Key=path)
        content = key['Body'].read()
        client.delete_object(Bucket=bucket, Key=path)
    else:
        import boto
        # assuming boto=2.2.2
        bucket = boto.connect_s3().get_bucket(bucket, validate=False)
        key = bucket.get_key(path)
        content = key.get_contents_as_string()
        bucket.delete_key(path)
    return (content, key) if with_key else content 

def _s3_conn(context):
    region = CONF.s3_region
    ec2_creds = clients.keystone(context).ec2.list(context.user_id)

    # Here we a) disable user's default config to let ec2api works independetly
    # of user's local settings;
    # b) specify region to be used by botocore;
    # c) do not change standard botocore keys to get these settings
    # from environment
    connection_data = {
        'config_file': (None, 'AWS_CONFIG_FILE', None, None),
        'region': ('region', 'AWS_DEFAULT_REGION', region, None),
    }
    session = botocore.session.get_session(connection_data)
    return session.create_client(
        's3', region_name=region, endpoint_url=CONF.s3_url,
        aws_access_key_id=ec2_creds[0].access,
        aws_secret_access_key=ec2_creds[0].secret,
        config=botocore.config.Config(signature_version='s3v4')) 

def get_session(self, region_name, role_arn):
        """
        Assumes the given role and returns a session object assuming said role.
        """
        session = botocore.session.get_session()
        if region_name is not None:
            session.set_config_variable('region', region_name)

        if role_arn is not None:
            sts = session.create_client(AUTH_SERVICE, region_name=region_name)
            credentials_dict = sts.assume_role(
                RoleArn=role_arn,
                RoleSessionName='EKSGetTokenAuth'
            )['Credentials']

            session.set_credentials(credentials_dict['AccessKeyId'],
                                    credentials_dict['SecretAccessKey'],
                                    credentials_dict['SessionToken'])
            return session
        else:
            return session 

def test_ddb_table_name():
    ddb = session.create_client('dynamodb', region_name='us-west-2')
    response = {
        'ResponseMetadata': {
            'RequestId': REQUEST_ID,
            'HTTPStatusCode': 403,
        }
    }

    with Stubber(ddb) as stubber:
        stubber.add_response('describe_table', response, {'TableName': 'mytable'})
        ddb.describe_table(TableName='mytable')

    subsegment = xray_recorder.current_segment().subsegments[0]
    assert subsegment.error
    assert subsegment.http['response']['status'] == 403

    aws_meta = subsegment.aws
    assert aws_meta['table_name'] == 'mytable'
    assert aws_meta['request_id'] == REQUEST_ID
    assert aws_meta['region'] == 'us-west-2'
    assert aws_meta['operation'] == 'DescribeTable' 

def __init__(self, uri, access_key=None, secret_key=None, acl=None):
        # BEGIN Backward compatibility for initialising without keys (and
        # without using from_crawler)
        no_defaults = access_key is None and secret_key is None
        if no_defaults:
            from scrapy.utils.project import get_project_settings
            settings = get_project_settings()
            if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
                import warnings
                from scrapy.exceptions import ScrapyDeprecationWarning
                warnings.warn(
                    "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
                    "without AWS keys is deprecated. Please supply credentials or "
                    "use the `from_crawler()` constructor.",
                    category=ScrapyDeprecationWarning,
                    stacklevel=2
                )
                access_key = settings['AWS_ACCESS_KEY_ID']
                secret_key = settings['AWS_SECRET_ACCESS_KEY']
        # END Backward compatibility
        u = urlparse(uri)
        self.bucketname = u.hostname
        self.access_key = u.username or access_key
        self.secret_key = u.password or secret_key
        self.is_botocore = is_botocore()
        self.keyname = u.path[1:]  # remove first "/"
        self.acl = acl
        if self.is_botocore:
            import botocore.session
            session = botocore.session.get_session()
            self.s3_client = session.create_client(
                's3', aws_access_key_id=self.access_key,
                aws_secret_access_key=self.secret_key)
        else:
            import boto
            self.connect_s3 = boto.connect_s3 

def _get_client(service_name):
    aws_access_key_id = _id
    aws_secret_key = _key
    if service_name in _api_clients:
        return _api_clients[service_name]

    session = _get_botocore_session()
    if service_name == 'elasticbeanstalk':
        endpoint_url = _endpoint_url
    else:
        endpoint_url = None
    try:
        LOG.debug('Creating new Botocore Client for ' + str(service_name))
        client = session.create_client(service_name,
                                       endpoint_url=endpoint_url,
                                       aws_access_key_id=aws_access_key_id,
                                       aws_secret_access_key=aws_secret_key,
                                       verify=_verify_ssl,
                                       config=Config(signature_version='s3v4'))

    except botocore.exceptions.ProfileNotFound as e:
        raise InvalidProfileError(e)
    LOG.debug('Successfully created session for ' + service_name)

    _api_clients[service_name] = client
    return client 

def test_list_parameter_counting():
    """
    Test special parameters that have shape of list are recorded
    as count based on `para_whitelist.json`
    """
    sqs = session.create_client('sqs', region_name='us-west-2')
    queue_urls = ['url1', 'url2']
    queue_name_prefix = 'url'
    response = {
        'QueueUrls': queue_urls,
        'ResponseMetadata': {
            'RequestId': '1234',
            'HTTPStatusCode': 200,
        }
    }

    with Stubber(sqs) as stubber:
        stubber.add_response('list_queues', response, {'QueueNamePrefix': queue_name_prefix})
        sqs.list_queues(QueueNamePrefix='url')

    subsegment = xray_recorder.current_segment().subsegments[0]
    assert subsegment.http['response']['status'] == 200

    aws_meta = subsegment.aws
    assert aws_meta['queue_count'] == len(queue_urls)
    # all whitelisted input parameters will be converted to snake case
    # unless there is an explicit 'rename_to' attribute in json key
    assert aws_meta['queue_name_prefix'] == queue_name_prefix 

def test_s3_bucket_name_capture():
    s3 = session.create_client('s3', region_name='us-west-2')
    response = {
        'ResponseMetadata': {
            'RequestId': REQUEST_ID,
            'HTTPStatusCode': 200,
        }
    }

    bucket_name = 'mybucket'

    with Stubber(s3) as stubber:
        stubber.add_response('list_objects_v2', response, {'Bucket': bucket_name})
        s3.list_objects_v2(Bucket=bucket_name)

    subsegment = xray_recorder.current_segment().subsegments[0]
    aws_meta = subsegment.aws

    assert aws_meta['bucket_name'] == bucket_name
    assert aws_meta['request_id'] == REQUEST_ID
    assert aws_meta['region'] == 'us-west-2'
    assert aws_meta['operation'] == 'ListObjectsV2' 

def _create_xray_client(self, ip='127.0.0.1', port='2000'):
        session = botocore.session.get_session()
        url = 'http://%s:%s' % (ip, port)
        return session.create_client('xray', endpoint_url=url,
                                     region_name='us-west-2',
                                     config=Config(signature_version=UNSIGNED),
                                     aws_access_key_id='', aws_secret_access_key=''
                                     ) 

def get_client(profile, region, service):
    """Returns a new or cached botocore service client for the AWS profile, region, and service.

    Warns when a service is not available for a region, which means we
    need to update botocore or skip that call for that region.
    """
    session = get_session(profile)

    if region not in session.get_available_regions(service):
        warnings.warn("service {} not available in {}".format(service, region))

    return session.create_client(service, region_name=region) 

def __init__(self, uri, access_key=None, secret_key=None, acl=None):
        # BEGIN Backward compatibility for initialising without keys (and
        # without using from_crawler)
        no_defaults = access_key is None and secret_key is None
        if no_defaults:
            from scrapy.utils.project import get_project_settings
            settings = get_project_settings()
            if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
                import warnings
                from scrapy.exceptions import ScrapyDeprecationWarning
                warnings.warn(
                    "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
                    "without AWS keys is deprecated. Please supply credentials or "
                    "use the `from_crawler()` constructor.",
                    category=ScrapyDeprecationWarning,
                    stacklevel=2
                )
                access_key = settings['AWS_ACCESS_KEY_ID']
                secret_key = settings['AWS_SECRET_ACCESS_KEY']
        # END Backward compatibility
        u = urlparse(uri)
        self.bucketname = u.hostname
        self.access_key = u.username or access_key
        self.secret_key = u.password or secret_key
        self.is_botocore = is_botocore()
        self.keyname = u.path[1:]  # remove first "/"
        self.acl = acl
        if self.is_botocore:
            import botocore.session
            session = botocore.session.get_session()
            self.s3_client = session.create_client(
                's3', aws_access_key_id=self.access_key,
                aws_secret_access_key=self.secret_key)
        else:
            import boto
            self.connect_s3 = boto.connect_s3 

def create_graph(session: botocore.session.Session, service_list: list, output: io.StringIO = os.devnull,
                 debug=False) -> Graph:
    """Constructs a Graph object.

    Information about the graph as it's built will be written to the IO parameter `output`.
    """
    stsclient = session.create_client('sts')
    caller_identity = stsclient.get_caller_identity()
    dprint(debug, "Caller Identity: {}".format(caller_identity['Arn']))
    metadata = {
        'account_id': caller_identity['Account'],
        'pmapper_version': principalmapper.__version__
    }

    iamclient = session.create_client('iam')

    # Gather users and roles, generating a Node per user and per role
    nodes_result = get_unfilled_nodes(iamclient, output, debug)

    # Gather groups from current list of nodes (users), generate Group objects, attach to nodes in-flight
    groups_result = get_unfilled_groups(iamclient, nodes_result, output, debug)

    # Resolve all policies, generate Policy objects, attach to all groups and nodes
    policies_result = get_policies_and_fill_out(iamclient, nodes_result, groups_result, output, debug)

    # Determine which nodes are admins and update node objects
    update_admin_status(nodes_result, output, debug)

    # Generate edges, generate Edge objects
    edges_result = edge_identification.obtain_edges(session, service_list, nodes_result, output, debug)

    return Graph(nodes_result, edges_result, policies_result, groups_result, metadata) 

def _get_client(service_name):
    aws_access_key_id = _id
    aws_secret_key = _key
    if service_name in _api_clients:
        return _api_clients[service_name]

    session = _get_botocore_session()
    if service_name == 'elasticbeanstalk':
        endpoint_url = _endpoint_url
    else:
        endpoint_url = None
    try:
        LOG.debug('Creating new Botocore Client for ' + str(service_name))
        client = session.create_client(service_name,
                                       endpoint_url=endpoint_url,
                                       aws_access_key_id=aws_access_key_id,
                                       aws_secret_access_key=aws_secret_key,
                                       verify=_verify_ssl,
                                       config=Config(signature_version='s3v4'))

    except botocore.exceptions.ProfileNotFound as e:
        raise InvalidProfileError(e)
    LOG.debug('Successfully created session for ' + service_name)

    _api_clients[service_name] = client
    return client 

def __init__(self, service, operation, region_name, endpoint_url=None, session=None,
                 connect_timeout=None, request_timeout=None):
        # set credentials manually
        session = session or botocore.session.get_session()
        # get_session accepts access_key, secret_key
        self.client = session.create_client(
            service,
            region_name=region_name,
            endpoint_url=endpoint_url
        )
        try:
            self.endpoint = self.client.endpoint
        except AttributeError:
            self.endpoint = self.client._endpoint

        self.operation = operation
        self.http_client = AsyncHTTPClient()

        self.proxy_host = None
        self.proxy_port = None
        https_proxy = getproxies_environment().get('https')
        if https_proxy:
            self._enable_curl_httpclient()

            proxy_parts = https_proxy.split(':')
            if len(proxy_parts) == 2 and proxy_parts[-1].isdigit():
                self.proxy_host, self.proxy_port = proxy_parts
                self.proxy_port = int(self.proxy_port)
            else:
                proxy = urlparse(https_proxy)
                self.proxy_host = proxy.hostname
                self.proxy_port = proxy.port

        self.request_timeout = request_timeout
        self.connect_timeout = connect_timeout 

def __init__(self):
        self.awslambda = boto3.client('lambda')
        session = botocore.session.get_session()
        self.events = session.create_client('events') 

def setUp(self):
        """Setup users, projects, and start a test server."""
        super(S3APITestCase, self).setUp()
        tempdir = self.useFixture(fixtures.TempDir())
        conf = self.useFixture(config_fixture.Config())
        conf.config(buckets_path=tempdir.path,
                    s3_listen='127.0.0.1',
                    s3_listen_port=0)

        self.server = s3server.get_wsgi_server()
        # NOTE(ft): this requires eventlet.monkey_patch, which is called in
        # tests/unit/__init__.py. Remove it out from there if you get these
        # tests rid of server run
        self.server.start()
        self.addCleanup(self.server.stop)

        s3_url = 'http://' + CONF.s3_listen + ':' + str(self.server.port)
        region = 'FakeRegion'
        connection_data = {
            'config_file': (None, 'AWS_CONFIG_FILE', None, None),
            'region': ('region', 'BOTO_DEFAULT_REGION', region, None),
        }
        session = botocore.session.get_session(connection_data)
        conn = session.create_client(
            's3', region_name=region, endpoint_url=s3_url,
            aws_access_key_id='fake', aws_secret_access_key='fake')
        self.conn = conn

        def get_http_connection(*args):
            """Get a new S3 connection, don't attempt to reuse connections."""
            return self.conn.new_http_connection(*args)

        self.conn.get_http_connection = get_http_connection