Python boto.vpc() Examples

def get_sg_subnets(name, vpc_id, region, profile=None):
    """
    Args:
        name (str): The name of the security group you are looking for.
        vpc_id (str): The VPC id where this security group resides.
        region (str): The AWS region.

    Basic Usage:
        >>> name = 'ProductionELB'
        >>> region = 'us-west-2'
        >>> vpc_id = 'vpc-123456'
        >>> subnets = get_sg_subnets(name, vpc_id, region)
        >>> print subnets

    Returns:
        String
    """
    cidrs = get_sg_cidrs(name, vpc_id, region, profile)
    subnets = list()
    for subnet in cidrs:
        subnets.append(subnet.split("/")[0])
    return subnets 

def vpc(self):
        """
            fetch vpc connection, open if necessary
        """
        if self._region['vpc_conn']:
            return self._region['vpc_conn']
        region = self.region()
        if self._region['vpc_conn'] is None:
            log.debug("Connecting to VPC region {}".format(region))
            self._region['vpc_conn'] = boto.vpc.connect_to_region(
                region,
                is_secure=False
            )
        assert self._region['vpc_conn'] is not None, (
               ("Failed to connect to VPC service in region {!r}"
                .format(region)))
        return self._region['vpc_conn'] 

def _get_net_info(self, server, resources):
        """
        Look up the IP address, gateway, and subnet range. 
        """
        # gw_type, gw_info = self._get_nat_info(server["vpc"], server["subnet"])
        # gw = gw_info["nics"][0]["ip_address"]
        # gw = "0.0.0.0"
        cidr = server["cidr"].split("/")[1]
        ip = self._get_data_ip(server)
        gw_info = server["cidr"].split("/")[0].split(".")
        gw = "%s.%s.%s.1" % (gw_info[0], gw_info[1], gw_info[2])

        # We want to use the host NIC, so modify LXC to use phys networking, and
        # then start the docker containers on the server. 
        lxc_opts = ["lxc.network.type = phys",
                    "lxc.network.ipv4 = %s/%s" % (ip, cidr),
                    "lxc.network.ipv4.gateway = %s" % gw,
                    "lxc.network.link = eth1",
                    "lxc.network.name = eth1", 
                    "lxc.network.flags = up"]
        return lxc_opts, ip 

def __init__(self, region):

        self.uid='ut_{}'.format(random.randint(1, 10000000000))
        self.region_name = region
        self.ec2_conn = boto.ec2.connect_to_region(region)
        self.vpc_conn = boto.vpc.connect_to_region(region)

        self.key_name = self.uid+'_kp' 
        
        self.vpc_id = ''
        self.vpc_name = self.uid+'_vpc'
        self.vpc_netmask = '10.0.0.0/24'

        self.subnet_id = ''
        self.subnet_name = self.uid+'_subnet'
        self.subnet_netmask = self.vpc_netmask 

def vpc_exists(name, region):
    """
    Args:
        name (str): The name of the vpc you are retrieving the id for.
        region (str): The AWS region.

    Basic Usage:
        >>> vpc_name = 'test'
        >>> aws_region = 'us-west-2'
        >>> vpc_id = vpc_exists(vpc_name, aws_region)
        'vpc-1234567'

    Returns:
        VPC ID
    """
    vpc_id = None
    try:
        vpc_id = get_vpc_id_by_name(name, region)
    except Exception:
        vpc_id = 'does not exist'
    return vpc_id 

def get_vpc_ids_from_names(vpc_names, region=None, profile=None):
    """Return a list of vpc ids from the list of vpc names that were matched.
    Args:
        vpc_names (list): List of vpc names you are searching for.
        client (Boto3.Client): The instantiated boto3 client.
    """
    vpc_ids = list()
    client = aws_client(region, 'ec2', profile)
    vpcs = client.describe_vpcs()
    for vpc in vpcs['Vpcs']:
        if 'Tags' in vpc:
            for tag in vpc['Tags']:
                if tag['Key'] == 'Name':
                    for name in vpc_names:
                        if re.search(name, tag['Value'], re.IGNORECASE):
                            vpc_ids.append(vpc['VpcId'])
    return vpc_ids 

def get_all_route_table_ids_except_vpc_names(vpc_names, region=None,
                                             profile=None):
    """
    Args:
        vpc_names (list): List of vpc names you are searching for.

    Kwargs:
        region (str): The AWS region.

    Basic Usage:
        >>> vpc_names = ['test', 'foo']
        >>> get_all_route_table_ids_except_vpc_names(vpc_names)
        ['rtb-123456']

    Returns:
        List of route table ids
    """
    route_ids = list()
    client = aws_client(region, 'ec2', profile)
    vpc_ids = get_vpc_ids_from_names(vpc_names, region, profile)
    params = {
        'Filters': [
            {
                'Name': 'association.main',
                'Values': ['false']
            }
        ]
    }
    routes = client.describe_route_tables(**params)
    if routes:
        for route in routes['RouteTables']:
            if route['VpcId'] not in vpc_ids:
                route_ids.append(route['RouteTableId'])
        if len(route_ids) > 0:
            return route_ids
        else:
            raise errors.AnsibleFilterError("No routes were found")
    else:
        raise errors.AnsibleFilterError("No routes were found") 

def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs):
    """
    :type aws_access_key_id: string
    :param aws_access_key_id: Your AWS Access Key ID

    :type aws_secret_access_key: string
    :param aws_secret_access_key: Your AWS Secret Access Key

    :rtype: :class:`boto.vpc.VPCConnection`
    :return: A connection to VPC
    """
    from boto.vpc import VPCConnection
    return VPCConnection(aws_access_key_id, aws_secret_access_key, **kwargs) 

def get_sg(name, vpc_id, region, profile=None):
    """
    Args:
        name (str): The name of the security group you are looking for.
        vpc_id (str): The VPC id where this security group resides.
        region (str): The AWS region.

    Basic Usage:
        >>> name = 'ProductionELB'
        >>> region = 'us-west-2'
        >>> vpc_id = 'vpc-123456'
        >>> security_group_id = get_sg(name, vpc_id, region)
        >>> print security_group_id

    Returns:
        String
    """
    connect = boto.ec2.connect_to_region(region)
    filter_by = {
        "tag-key": "Name",
        "tag-value": name,
        "vpc-id": vpc_id
    }
    try:
        sg_groups = connect.get_all_security_groups(filters=filter_by)
        if len(sg_groups) == 1:
            return sg_groups[0].id
        elif len(sg_groups) > 1:
            raise errors.AnsibleFilterError(
                "Too many results for {0}: {1}".format(
                    name, ",".join(sg_groups)
                )
            )
        else:
            raise errors.AnsibleFilterError(
                "Security Group {0} was not found".format(name)
            )
    except botocore.exceptions.ClientError as e:
        raise e 

def _inspect_instance(self, instance_id):
        instances = self.ec2.get_only_instances(instance_ids=[instance_id])

        instance_info = { 'nics': [] }
        for eni in instances[0].interfaces:
            _filter = { "network-interface-id" : eni.id }
            addrs = self.ec2.get_all_addresses(filters=_filter)
            subnets = self.vpc.get_all_subnets(subnet_ids=[eni.subnet_id])

            # Capture global network info.
            instance_info['vpc'] = eni.vpc_id
            instance_info['subnet'] = eni.subnet_id
            instance_info['cidr'] = subnets[0].cidr_block

            # Capture NIC specific information. 
            if addrs and len(addrs) > 0:
                for a in addrs:
                    logging.warning("USING PUBLIC ADDR:" + a.public_ip)
                    instance_info["nics"].append( { "ip_address" : a.private_ip_address,
                                             "floating_ip" : a.public_ip,
                                             "index" : eni.attachment.device_index, 
                                             "eni" : eni.id } )
            else:
                nic_info = { "ip_address" : eni.private_ip_address,
                             "index" : eni.attachment.device_index, 
                             "eni" : eni.id }

                # Sometimes the instance has a public IP automatically
                # assigned to it, but it's only for eth0. 
                if eni.attachment.device_index == 0:
                    nic_info["floating_ip"] = instances[0].ip_address

                instance_info["nics"].append( nic_info )
        return instance_info 

def _collect_subnet_info(self, vpc_id):
        subnets = self.vpc.get_all_subnets()
        for subnet in subnets:
            if subnet.vpc_id == vpc_id:
                self.subnets.append( { "Subnet:" + subnet.id : { 'cidr' : subnet.cidr_block }} )

                # Keep track of the subnet CIDRs. Check for the data and
                # manage subnets independently since they may actually
                # be the exact same subnet (user can specify this in configuration). 
                if self.data_subnet == subnet.id:
                    self.data_cidr = subnet.cidr_block

                if self.manage_subnet == subnet.id:
                    self.manage_cidr = subnet.cidr_block 

def _collect_vpc_info(self, vpc_id):
        vpcs = self.vpc.get_all_vpcs(vpc_ids=[vpc_id])
        for vpc in vpcs:
            return vpc.cidr_block 

def _create_subnet_plan(self, subnet_name, vpc, is_ref):
        plan = { "AWSTemplateFormatVersion" : "2010-09-09",
                 "Description" : "Ferry generated Heat plan",
                 "Resources" : {} }

        # Determine whether we're creating a VPC that we've just created
        # or one that already exists. 
        if is_ref:
            network = { "Ref" : vpc }
        else:
            network = vpc

        public_subnet, subnet_desc = self._create_subnet(subnet_name, network)
        plan["Resources"] = public_subnet
        return plan, subnet_desc 

def _create_floatingip_plan(self, cluster_uuid, instances):
        """
        Assign an elastic IP to the secondary device of each instance. 
        """
        plan = { "AWSTemplateFormatVersion" : "2010-09-09",
                 "Description" : "Ferry generated CloudFormation plan",
                 "Resources" :  {} }
        desc = {}
        for instance in instances:
            eip_name = "FerryEIP" + instance["name"]
            assoc_name = "FerryEIPAssoc" + instance["name"]
            eip_resource = {
                "Type" : "AWS::EC2::EIP",
                "Properties" : {
                    "Domain" : "vpc"
                }
            }
            assoc_resource = {
                "Type": "AWS::EC2::EIPAssociation",
                "Properties": {
                    "AllocationId" : { "Fn::GetAtt" : [ eip_name, "AllocationId" ]},
                    "NetworkInterfaceId": { "Ref" : instance["data_nic"] }
                    # "InstanceId": { "Ref" : instance["name"] }
                }
            }
            plan["Resources"][eip_name] = eip_resource
            plan["Resources"][assoc_name] = assoc_resource
            desc[eip_name] = { "type" : "AWS::EC2::EIP" }

        return plan, desc 

def _create_routetable(self, name, subnet, vpc):
        plan = { name : { "Type" : "AWS::EC2::RouteTable",
                          "Properties" : { "VpcId" : vpc}}}
        desc = { name : { "type" : "AWS::EC2::RouteTable",
                          "vpc" : vpc }}
        return plan, desc 

def _init_aws_clients(self):
        # We will need to use both EC2 and VPC. 
        self.ec2 = boto.ec2.connect_to_region(self.default_dc,
                                              aws_access_key_id = self.aws_access_key,
                                              aws_secret_access_key = self.aws_secret_key)
        self.vpc = boto.vpc.VPCConnection(aws_access_key_id = self.aws_access_key,
                                          aws_secret_access_key = self.aws_secret_key)
        self.cf = boto.cloudformation.connect_to_region(self.default_dc, 
                                                        aws_access_key_id = self.aws_access_key,
                                                        aws_secret_access_key = self.aws_secret_key) 

def vpc(self):
        """
        :rtype: VPCConnection
        """
        if self.__vpc is None:
            self.__vpc = self.__aws_connect(vpc)
        return self.__vpc

    # ec2 = vpc works, too, but confuses the type hinter in PyCharm 

def get_route_table_ids(vpc_id, region=None, profile=None):
    """
    Args:
        vpc_id (str): The vpc id in which the subnet you are looking
            for lives in,

    Basic Usage:
        >>> vpc_id = 'vpc-12345678'
        >>> get_route_table_ids(vpc_id)
        ['rtb-1234567a']

    Returns:
        List of route table ids
    """
    route_ids = list()
    client = aws_client(region, 'ec2', profile)
    params = {
        'Filters': [
            {
                'Name': 'vpc-id',
                'Values': [vpc_id],
            },
            {
                'Name': 'association.main',
                'Values': ['false']
            }
        ]
    }
    routes = client.describe_route_tables(**params)
    if routes:
        route_ids = (
            map(lambda route: route['RouteTableId'], routes['RouteTables'])
        )
        return route_ids
    else:
        raise errors.AnsibleFilterError("No routes were found") 

def get_subnet_ids_in_zone(vpc_id, zone, region=None, profile=None):
    """
    Args:
        vpc_id (str): The vpc id in which the subnet you are looking
            for lives in,
        zone (str): The region in which the subnet resides.

    Basic Usage:
        >>> vpc_id = 'vpc-12345678'
        >>> aws_region = 'us-west-2'
        >>> zone = 'us-west-2c'
        >>> get_subnet_ids_in_zone(vpc_id, zone, aws_region)
        [u'subnet-4324567', u'subnet-12345678', u'subnet-6543210']

    Returns:
        List of subnet ids
    """
    subnet_ids = list()
    client = aws_client(region, 'ec2', profile)
    params = {
        'Filters': [
            {
                'Name': 'vpc-id',
                'Values': [vpc_id],
            },
            {
                'Name': 'availabilityZone',
                'Values': [zone],
            }
        ]
    }
    subnets = client.describe_subnets(**params)['Subnets']
    if subnets:
        subnet_ids = map(lambda subnet: subnet['SubnetId'], subnets)
        return subnet_ids
    else:
        raise errors.AnsibleFilterError("No subnets were found") 

def get_all_route_table_ids_except(vpc_id, region=None, profile=None):
    """
    Args:
        vpc_id (str): The vpc you want to exclude routes from.

    Basic Usage:
        >>> vpc_id = 'vpc-98c797fd'
        >>> get_all_route_table_ids_except(vpc_id)
        ['rtb-5f78343a']

    Returns:
        List of route table ids
    """
    route_ids = list()
    client = aws_client(region, 'ec2', profile)
    params = {
        'Filters': [
            {
                'Name': 'association.main',
                'Values': ['false']
            }
        ]
    }
    routes = client.describe_route_tables(**params)
    if routes:
        for route in routes['RouteTables']:
            if route['VpcId'] != vpc_id:
                route_ids.append(route['RouteTableId'])
        if len(route_ids) > 0:
            return route_ids
        else:
            raise errors.AnsibleFilterError("No routes were found")
    else:
        raise errors.AnsibleFilterError("No routes were found") 

def _create_security_group(self, group_name, network, is_ref, ports, internal, outbound):
        """
        Create and assign a security group to the supplied server. 
        """

        # Determine whether we're creating a VPC that we've just created
        # or one that already exists. 
        if is_ref:
            vpc = { "Ref" : network }
        else:
            vpc = network

        # Create the basic security group. 
        # This only includes SSH. We can later update the group
        # to include additional ports. 
        desc = { group_name : { "Type" : "AWS::EC2::SecurityGroup",
                                "Properties" : { "GroupDescription" : "Ferry firewall rules", 
                                                 "VpcId" : vpc, 
                                                 "SecurityGroupIngress" : [ { "IpProtocol" : "tcp",
                                                                              "CidrIp": "0.0.0.0/0",
                                                                              "FromPort" : "22", 
                                                                              "ToPort" : "22" }]}}}
        # Ports for internal connections. 
        for p in ports:
            min_port = p[0]
            max_port = p[1]
            desc[group_name]["Properties"]["SecurityGroupIngress"].append({ "IpProtocol" : "tcp",
                                                                            "CidrIp": "0.0.0.0/0",
                                                                            "FromPort" : min_port, 
                                                                            "ToPort" : max_port })

        # Make all data subnet traffic open. This is necessary because many systems 
        # do things like open random ports for IPC. This is true even for connecting to clients. 
        desc[group_name]["Properties"]["SecurityGroupIngress"].append({ "IpProtocol" : "tcp",
                                                                        "CidrIp": self.data_cidr,
                                                                        "FromPort" : "0",
                                                                        "ToPort" : "65535" })

        # Also need to open up communication for the manage subnet. This will
        # allow connectors to freely communicate with the data nodes. 
        if self.manage_cidr != self.data_cidr:
            desc[group_name]["Properties"]["SecurityGroupIngress"].append({ "IpProtocol" : "tcp",
                                                                            "CidrIp": self.manage_cidr,
                                                                            "FromPort" : "0",
                                                                            "ToPort" : "65535" })

        # Outbound ports. This is not required. If the user
        # doesn't supply egress rules, then all outgoing requests are allowed. 
        if len(outbound) > 0:
            desc[group_name]["Properties"]["SecurityGroupEgress"] = []
        for p in outbound:
            min_port = p[0]
            max_port = p[1]
            desc[group_name]["Properties"]["SecurityGroupEgress"].append({ "IpProtocol" : "tcp",
                                                                           "CidrIp": "0.0.0.0/0",
                                                                           "FromPort" : min_port,
                                                                           "ToPort" : max_port })
        return desc 

def _create_nat_plan(self, table_name, public_subnet_id, public_subnet_name, private_subnet, vpc, is_ref):
        plan = { "AWSTemplateFormatVersion" : "2010-09-09",
                 "Description" : "Ferry generated Heat plan",
                 "Resources" : {} }

        # Create a NAT security group. This security group is
        # fairly locked down and only lets machines communicate via
        # HTTP and make outbound SSH calls.
        logging.info("creating NAT security group")
        sec_group_name = "FerryNATSec%s" % private_subnet
        sec_group_plan = self._create_security_group(group_name = sec_group_name, 
                                                     network = vpc,
                                                     is_ref = is_ref, 
                                                     ports = [("80","80"),("443","443")],
                                                     internal = [],
                                                     outbound = [("80","80"),("443","443")])

        # Create the NAT instance. Thsi is the actual machine that
        # handles all the NAT requests. 
        if public_subnet_id:
            public_subnet = public_subnet_id
        else:
            public_subnet = { "Ref" : public_subnet_name }
        instance_name = "FerryNAT%s" % private_subnet
        instance_plan, instance_desc = self._create_instance(name = instance_name, 
                                                             subnet = public_subnet,
                                                             image = self.nat_image, 
                                                             size = self.default_personality,
                                                             sec_group = sec_group_name,
                                                             user_data = None, 
                                                             single_network = True)

        # Also create a new route associated with this NAT. 
        # This will send all outbound internet traffic to the gateway. 
        route_plan = { "FerryRoute" + private_subnet: { "Type" : "AWS::EC2::Route",
                                                        "Properties" : { "InstanceId" : { "Ref" : instance_name },
                                                                         "RouteTableId" : { "Ref" : table_name },
                                                                         "DestinationCidrBlock" : "0.0.0.0/0" }}}
        plan["Resources"] = dict(instance_plan.items() + 
                                 route_plan.items() + 
                                 sec_group_plan.items())
        desc = instance_desc
        return plan, desc 

def _create_igw_plan(self, igw_name, igw_id, route_table, vpc, is_ref):
        """
        Create a new internet gateway and associate it 
        with the VPC. 
        """

        # Determine whether we're creating a VPC that we've just created
        # or one that already exists. 
        if is_ref:
            network = { "Ref" : vpc }
        else:
            network = vpc

        # Create a new internet gateway. This one is pretty simple
        # since there are no options ;)
        if not igw_id:
            logging.info("creating IGW")
            igw_plan = { igw_name : { "Type" : "AWS::EC2::InternetGateway" }}
            name = { "Ref" : igw_name }
        else:
            logging.info("using IGW " + igw_id)
            # The user has supplied the IGW id, so we
            # shouldn't create one.
            igw_plan = {}
            name = igw_id
            
        # Attach the internet gateway to our VPC. 
        attach_plan = { igw_name + "Attach": { "Type" : "AWS::EC2::VPCGatewayAttachment",
                                               "Properties" : { "InternetGatewayId" : name,
                                                                "VpcId" : network}}}

        # Also create a new route associated with this
        # internet gateway. This will send all outbound internet
        # traffic to the gateway. 
        route_plan = { name + "Route": { "Type" : "AWS::EC2::Route",
                          "Properties" : { "GatewayId" : name,
                                           "RouteTableId" : { "Ref" : route_table },
                                           "DestinationCidrBlock" : "0.0.0.0/0" }}}

        desc = { "type" : "AWS::EC2::InternetGateway",
                 "vpc" : network }
        plan = { "AWSTemplateFormatVersion" : "2010-09-09",
                 "Description" : "Ferry generated Heat plan",
                 "Resources" : {} }
        plan["Resources"] = dict(igw_plan.items() + 
                                 attach_plan.items() +
                                 route_plan.items() )
        return plan, desc 

def get_all_vpcs_info_except(except_ids, region=None, profile=None):
    """
    Args:
        except_ids (list): List of vpcs, that you do not want to match against.

    Basic Usage:
        >>> vpc_ids = ['vpc-345621']
        >>> get_all_vpcs_info_except(vpc_ids)
        ['vpc-1234567', 'vpc-97654321']

    Returns:
        List of vpc ids
    """
    vpcs_info = list()
    client = aws_client(region, 'ec2', profile)
    params = {
        'Filters': [
            {
                'Name': 'state',
                'Values': ['available'],
            },
            {
                'Name': 'isDefault',
                'Values': ['false'],
            }
        ]
    }
    vpcs = client.describe_vpcs(**params)
    if vpcs:
        for vpc in vpcs['Vpcs']:
            if vpc['VpcId'] not in except_ids:
                name = ''
                if vpc.get('Tags', None):
                    for tag in vpc['Tags']:
                        if tag.get('Key', None) == 'Name':
                            name = tag.get('Value')

                    vpcs_info.append(
                        {
                            'name': name,
                            'id': vpc['VpcId'],
                            'cidr': vpc['CidrBlock'],
                        }
                    )
    if vpcs_info:
        return vpcs_info
    else:
        raise errors.AnsibleFilterError("No vpcs were found") 

def get_subnet_ids(vpc_id, cidrs=None, region=None, profile=None):
    """
    Args:
        vpc_id (str): The vpc id in which the subnet you are looking
            for lives in.
    Kwargs:
        cidrs (list): The list of cidrs that you are performing the search on.
        region (str): The AWS region.

    Basic Usage:
        >>> cidrs = ['10.100.10.0/24', '10.100.12.0/24', '10.100.11.0/24']
        >>> vpc_id = 'vpc-123456'
        >>> aws_region = 'us-west-2'
        >>> get_subnet_ids(vpc_id, cidrs, aws_region)
        [u'subnet-123456', u'subnet-765432', u'subnet-123456']

    Returns:
        List of subnet ids
    """
    subnet_ids = list()
    client = aws_client(region, 'ec2', profile)
    params = {
        'Filters': [
            {
                'Name': 'vpc-id',
                'Values': [vpc_id],
            }
        ]
    }
    if cidrs:
        params['Filters'].append(
            {
                'Name': 'cidrBlock',
                'Values': cidrs,
            }
        )
    subnets = (
        sorted(
            client.describe_subnets(**params)['Subnets'],
            key=lambda subnet: subnet['AvailabilityZone']
        )
    )
    if subnets:
        subnet_ids = map(lambda subnet: subnet['SubnetId'], subnets)
        return subnet_ids
    else:
        raise errors.AnsibleFilterError("No subnets were found") 

def get_sg_cidrs(name, vpc_id, region, profile=None):
    """
    Args:
        name (str): The name of the security group you are looking for.
        vpc_id (str): The VPC id where this security group resides.
        region (str): The AWS region.

    Basic Usage:
        >>> name = 'ProductionELB'
        >>> region = 'us-west-2'
        >>> vpc_id = 'vpc-123456'
        >>> security_group_id = get_sg(name, vpc_id, region)
        >>> print security_group_id

    Returns:
        String
    """
    client = aws_client(region, 'ec2', profile)
    params = {
        "Filters": [
            {
                "Name": "tag-key",
                "Values": ["Name"]
            },
            {
                "Name": "tag-value",
                "Values": [name]
            },
            {
                "Name": "vpc-id",
                "Values": [vpc_id],
            }
        ]
    }
    try:
        sg_groups = client.describe_security_groups(**params)['SecurityGroups']
        if len(sg_groups) == 1:
            cidrs = map(lambda x: x['CidrIp'], sg_groups[0]['IpPermissions'][0]['IpRanges'])
            return cidrs
        elif len(sg_groups) > 1:
            raise errors.AnsibleFilterError(
                "Too many results for {0}: {1}".format(
                    name, ",".join(sg_groups)
                )
            )
        else:
            raise errors.AnsibleFilterError(
                "Security Group {0} was not found".format(name)
            )
    except botocore.exceptions.ClientError as e:
        raise e