com.amazonaws.services.autoscaling.model.Tag Java Examples

/***
 * Get list of {@link AutoScalingGroup}s for a given tag
 *
 * @param tag Tag to filter the auto scaling groups
 * @return List of {@link AutoScalingGroup}s qualifying the filter tag
 */
public List<AutoScalingGroup> getAutoScalingGroupsWithTag(Tag tag) {

  final AmazonAutoScaling autoScaling = getAmazonAutoScalingClient();

  final DescribeAutoScalingGroupsRequest describeAutoScalingGroupsRequest = new DescribeAutoScalingGroupsRequest();

  final List<AutoScalingGroup> allAutoScalingGroups = autoScaling
      .describeAutoScalingGroups(describeAutoScalingGroupsRequest)
      .getAutoScalingGroups();

  final List<AutoScalingGroup> filteredAutoScalingGroups = Lists.newArrayList();
  for (AutoScalingGroup autoScalingGroup : allAutoScalingGroups) {
    for (TagDescription tagDescription : autoScalingGroup.getTags()) {
      if (tagDescription.getKey().equalsIgnoreCase(tag.getKey()) &&
          tagDescription.getValue().equalsIgnoreCase(tag.getValue())) {
        filteredAutoScalingGroups.add(autoScalingGroup);
      }
    }
  }

  return filteredAutoScalingGroups;
}
 

public void setDeployMetadataTags(final String version, Properties properties) {
    List<Tag> tags = new ArrayList<>();
    tags.add(new Tag().withPropagateAtLaunch(true)
            .withResourceType(AUTO_SCALING_GROUP)
            .withKey(activeConfiguration.getDeployType().getLatestRequestTag()).withValue(version)
            .withResourceId(activeConfiguration.getAutoScalingGroupId()));
    tags.add(new Tag().withPropagateAtLaunch(true)
            .withResourceType(AUTO_SCALING_GROUP)
            .withKey(activeConfiguration.getDeployType().getScopeTag()).withValue(Boolean.toString(activeConfiguration.isTestScope()))
            .withResourceId(activeConfiguration.getAutoScalingGroupId()));

    if (!activeConfiguration.getAutoScalingProperties().isEmpty()) {
        tags.add(new Tag().withPropagateAtLaunch(true)
                .withResourceType(AUTO_SCALING_GROUP)
                .withKey(activeConfiguration.getDeployType().getPropertiesTag()).withValue(activeConfiguration.getAutoScalingProperties().stream().map(key -> key + ":" + getProperty(key, properties)).collect(Collectors.joining(";")))
                .withResourceId(activeConfiguration.getAutoScalingGroupId())
        );
    }
    if (!activeConfiguration.getExclusions().isEmpty()) {
        tags.add(new Tag().withPropagateAtLaunch(true)
                .withResourceType(AUTO_SCALING_GROUP)
                .withKey(activeConfiguration.getDeployType().getExclusionTag()).withValue(activeConfiguration.getExclusions().stream().map(e -> e.getGroupId() + ":" + e.getArtifactId()).collect(Collectors.joining(";")))
                .withResourceId(activeConfiguration.getAutoScalingGroupId()));
    }
    awsAsClient.createOrUpdateTags(new CreateOrUpdateTagsRequest().withTags(tags));
}
 

@RequestMapping(value = "/autoscalinggroup", method = RequestMethod.POST)
@ResponseBody
public void createAutoScalingGroup(@RequestParam("accessKey") String accessKey, @RequestParam("secretKey") String secretKey,
        @RequestBody Map<String, String> body) {
	awsec2Service.createAmazonAutoScalingGroup(
               awsCredentialsFactory.createSimpleAWSCredentials(accessKey, secretKey),
	        body.get("autoScalingGroupName"),
               Integer.valueOf(body.get("maxSize")),
               Integer.valueOf(body.get("minSize")),
               Integer.valueOf(body.get("defaultCooldown")),
	        body.get("launchConfigurationName"),
               Collections.<String> emptyList(),
               Collections.<Tag> emptyList());
}
 

@VisibleForTesting
Optional<String> getReconnectableClusterId() throws IOException {
  // List ASGs with Tag of cluster name
  final Tag clusterNameTag = new Tag()
      .withKey(CLUSTER_NAME_ASG_TAG)
      .withValue(this.clusterName);
  final List<AutoScalingGroup> autoScalingGroups = this.awsSdkClient.getAutoScalingGroupsWithTag(clusterNameTag);

  // If no auto scaling group is found, we don't have an existing cluster to connect to
  if (autoScalingGroups.size() == 0) {
    return Optional.absent();
  }

  // If more than 0 auto scaling groups are found, validate the setup
  if (autoScalingGroups.size() != 2) {
    throw new IOException("Expected 2 auto scaling groups (1 each for master and workers) but found: " +
      autoScalingGroups.size());
  }

  // Retrieve cluster information from ASGs
  Optional<String> clusterId = Optional.absent();
  Optional<AutoScalingGroup> masterAsg = Optional.absent();
  Optional<AutoScalingGroup> workersAsg = Optional.absent();

  for (TagDescription tagDescription : autoScalingGroups.get(0).getTags()) {
    LOGGER.info("Found tag: " + tagDescription);
    if (tagDescription.getKey().equalsIgnoreCase(CLUSTER_ID_ASG_TAG)) {
      clusterId = Optional.of(tagDescription.getValue());
    }
    if (tagDescription.getKey().equalsIgnoreCase(ASG_TYPE_ASG_TAG)) {
      if (tagDescription.getValue().equalsIgnoreCase(ASG_TYPE_MASTER)) {
        masterAsg = Optional.of(autoScalingGroups.get(0));
        workersAsg = Optional.of(autoScalingGroups.get(1));
      } else {
        masterAsg = Optional.of(autoScalingGroups.get(1));
        workersAsg = Optional.of(autoScalingGroups.get(0));
      }
    }
  }

  if (!clusterId.isPresent()) {
    throw new IOException("Found 2 auto scaling group names for: " + this.clusterName +
        " but tags seem to be corrupted, hence could not determine cluster id");
  }

  if (!masterAsg.isPresent() || !workersAsg.isPresent()) {
    throw new IOException("Found 2 auto scaling group names for: " + this.clusterName +
    " but tags seem to be corrupted, hence could not determine master and workers ASG");
  }

  // Get Master and Workers launch config name and auto scaling group name
  this.masterAutoScalingGroupName = masterAsg.get().getAutoScalingGroupName();
  this.masterLaunchConfigName = masterAsg.get().getLaunchConfigurationName();
  this.workerAutoScalingGroupName = workersAsg.get().getAutoScalingGroupName();
  this.workerLaunchConfigName = workersAsg.get().getLaunchConfigurationName();

  LOGGER.info("Trying to find cluster master public ip");
  this.masterPublicIp = getMasterPublicIp();
  LOGGER.info("Master public ip: "+ this.masterPublicIp);

  return clusterId;
}
 

private String launchClusterMaster(String uuid, String keyName, String securityGroups,
    AvailabilityZone availabilityZone) {

  // Get cloud-init script to launch cluster master
  final String userData = CloudInitScriptBuilder.buildClusterMasterCommand(this.clusterName,
      this.nfsParentDir,
      this.sinkLogRootDir,
      this.awsConfDir,
      this.appWorkDir,
      this.masterS3ConfUri,
      this.masterS3ConfFiles,
      this.masterS3JarsUri,
      this.masterS3JarsFiles,
      this.masterJarsDir,
      this.masterJvmMemory,
      this.masterJvmArgs,
      this.gobblinVersion);

  // Create launch config for Cluster master
  this.masterLaunchConfigName = MASTER_LAUNCH_CONFIG_NAME_PREFIX + uuid;
  this.awsSdkClient.createLaunchConfig(this.masterLaunchConfigName,
      this.masterAmiId,
      this.masterInstanceType,
      keyName,
      securityGroups,
      Optional.<String>absent(),
      Optional.<String>absent(),
      Optional.<BlockDeviceMapping>absent(),
      Optional.<String>absent(),
      Optional.<InstanceMonitoring>absent(),
      userData);

  // Create ASG for Cluster master
  // TODO: Make size configurable when we have support multi-master
  this.masterAutoScalingGroupName = MASTER_ASG_NAME_PREFIX + uuid;
  final int minNumMasters = 1;
  final int maxNumMasters = 1;
  final int desiredNumMasters = 1;
  final Tag clusterNameTag = new Tag().withKey(CLUSTER_NAME_ASG_TAG).withValue(this.clusterName);
  final Tag clusterUuidTag = new Tag().withKey(CLUSTER_ID_ASG_TAG).withValue(uuid);
  final Tag asgTypeTag = new Tag().withKey(ASG_TYPE_ASG_TAG).withValue(ASG_TYPE_MASTER);
  this.awsSdkClient.createAutoScalingGroup(this.masterAutoScalingGroupName,
      this.masterLaunchConfigName,
      minNumMasters,
      maxNumMasters,
      desiredNumMasters,
      Optional.of(availabilityZone.getZoneName()),
      Optional.<Integer>absent(),
      Optional.<Integer>absent(),
      Optional.<String>absent(),
      Optional.<String>absent(),
      Optional.<String>absent(), Lists.newArrayList(clusterNameTag, clusterUuidTag, asgTypeTag));

  LOGGER.info("Waiting for cluster master to launch");
  this.masterPublicIp = getMasterPublicIp();
  LOGGER.info("Master public ip: "+ this.masterPublicIp);

  return uuid;
}
 

private void launchWorkUnitRunners(String uuid, String keyName,
    String securityGroups,
    AvailabilityZone availabilityZone) {

  // Get cloud-init script to launch cluster worker
  final String userData = CloudInitScriptBuilder.buildClusterWorkerCommand(this.clusterName,
      this.nfsParentDir,
      this.sinkLogRootDir,
      this.awsConfDir,
      this.appWorkDir,
      this.masterPublicIp,
      this.workerS3ConfUri,
      this.workerS3ConfFiles,
      this.workerS3JarsUri,
      this.workerS3JarsFiles,
      this.workerJarsDir,
      this.workerJvmMemory,
      this.workerJvmArgs,
      this.gobblinVersion);

  // Create launch config for Cluster worker
  this.workerLaunchConfigName = WORKERS_LAUNCH_CONFIG_PREFIX + uuid;
  this.awsSdkClient.createLaunchConfig(this.workerLaunchConfigName,
      this.workerAmiId,
      this.workerInstanceType,
      keyName,
      securityGroups,
      Optional.<String>absent(),
      Optional.<String>absent(),
      Optional.<BlockDeviceMapping>absent(),
      Optional.<String>absent(),
      Optional.<InstanceMonitoring>absent(),
      userData);

  // Create ASG for Cluster workers
  this.workerAutoScalingGroupName = WORKERS_ASG_NAME_PREFIX + uuid;
  final Tag clusterNameTag = new Tag().withKey(CLUSTER_NAME_ASG_TAG).withValue(this.clusterName);
  final Tag clusterUuidTag = new Tag().withKey(CLUSTER_ID_ASG_TAG).withValue(uuid);
  final Tag asgTypeTag = new Tag().withKey(ASG_TYPE_ASG_TAG).withValue(ASG_TYPE_WORKERS);
  this.awsSdkClient.createAutoScalingGroup(this.workerAutoScalingGroupName,
      this.workerLaunchConfigName,
      this.minWorkers,
      this.maxWorkers,
      this.desiredWorkers,
      Optional.of(availabilityZone.getZoneName()),
      Optional.<Integer>absent(),
      Optional.<Integer>absent(),
      Optional.<String>absent(),
      Optional.<String>absent(),
      Optional.<String>absent(),
      Lists.newArrayList(clusterNameTag, clusterUuidTag, asgTypeTag));
}
 

/***
 * Create and launch an {@link AmazonAutoScaling} group
 *
 * @param groupName Auto scaling group name
 * @param launchConfig Launch configuration string
 * @param minSize Minimum number of instances to maintain in auto scaling group
 * @param maxSize Maximum number of instances to scale up-to for load
 * @param desiredCapacity Desired number of instances to maintain in auto scaling group
 * @param availabilityZones Optional availability zones to make use of
 * @param cooldown Optional cooldown period before any scaling event (default is 300 secs)
 * @param healthCheckGracePeriod Optional grace period till which no health check is performed after bootup (default is 300 secs)
 * @param healthCheckType Optional health check type (default is EC2 instance check)
 * @param loadBalancer Optional load balancer to use
 * @param terminationPolicy Optional termination policies
 * @param tags Optional tags to set on auto scaling group (they are set to propagate to EC2 instances implicitly)
 */
public void createAutoScalingGroup(String groupName,
    String launchConfig,
    Integer minSize, Integer maxSize, Integer desiredCapacity,
    Optional<String> availabilityZones,
    Optional<Integer> cooldown,
    Optional<Integer> healthCheckGracePeriod,
    Optional<String> healthCheckType,
    Optional<String> loadBalancer,
    Optional<String> terminationPolicy,
    List<Tag> tags) {

  AmazonAutoScaling autoScaling = getAmazonAutoScalingClient();

  // Propagate ASG tags to EC2 instances launched under the ASG by default
  // (we want to ensure this, hence not configurable)
  final List<Tag> tagsWithPropagationSet = Lists.newArrayList();
  for (Tag tag : tags) {
    tagsWithPropagationSet.add(tag.withPropagateAtLaunch(true));
  }

  CreateAutoScalingGroupRequest createAutoScalingGroupRequest = new CreateAutoScalingGroupRequest()
      .withAutoScalingGroupName(groupName)
      .withLaunchConfigurationName(launchConfig)
      .withMinSize(minSize)
      .withMaxSize(maxSize)
      .withDesiredCapacity(desiredCapacity)
      .withTags(tagsWithPropagationSet);
  if (availabilityZones.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withAvailabilityZones(SPLITTER.splitToList(availabilityZones.get()));
  }
  if (cooldown.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withDefaultCooldown(cooldown.get());
  }
  if (healthCheckGracePeriod.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withHealthCheckGracePeriod(healthCheckGracePeriod.get());
  }
  if (healthCheckType.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withHealthCheckType(healthCheckType.get());
  }
  if (loadBalancer.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withLoadBalancerNames(SPLITTER.splitToList(loadBalancer.get()));
  }
  if (terminationPolicy.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withTerminationPolicies(SPLITTER.splitToList(terminationPolicy.get()));
  }

  autoScaling.createAutoScalingGroup(createAutoScalingGroupRequest);

  LOGGER.info("Created AutoScalingGroup: " + groupName);
}
 

@BeforeClass
public void setUp() throws Exception {

  // Mock AWS SDK calls
  MockitoAnnotations.initMocks(this);

  PowerMockito.whenNew(AWSSdkClient.class).withAnyArguments().thenReturn(awsSdkClient);

  Mockito.doNothing()
      .when(awsSdkClient)
      .createSecurityGroup(Mockito.anyString(), Mockito.anyString());
  Mockito.doReturn(Lists.<AvailabilityZone>newArrayList(availabilityZone))
      .when(awsSdkClient)
      .getAvailabilityZones();
  Mockito.doReturn("dummy")
      .when(awsSdkClient)
      .createKeyValuePair(Mockito.anyString());
  Mockito.doReturn(Lists.<AutoScalingGroup>newArrayList(masterASG, workerASG))
      .when(awsSdkClient)
      .getAutoScalingGroupsWithTag(Mockito.any(Tag.class));
  Mockito.doReturn(Lists.<Instance>newArrayList(instance))
      .when(awsSdkClient)
      .getInstancesForGroup(Mockito.anyString(), Mockito.anyString());
  Mockito.doReturn(Lists.<S3ObjectSummary>newArrayList())
      .when(awsSdkClient)
      .listS3Bucket(Mockito.anyString(), Mockito.anyString());
  Mockito.doNothing()
      .when(awsSdkClient)
      .addPermissionsToSecurityGroup(Mockito.any(String.class), Mockito.any(String.class), Mockito.any(String.class),
          Mockito.any(Integer.class), Mockito.any(Integer.class));
  Mockito.doNothing()
      .when(awsSdkClient)
      .createAutoScalingGroup(Mockito.any(String.class), Mockito.any(String.class), Mockito.any(Integer.class),
          Mockito.any(Integer.class), Mockito.any(Integer.class), Mockito.any(Optional.class),
          Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class),
          Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(List.class));
  Mockito.doNothing()
      .when(awsSdkClient)
      .createLaunchConfig(Mockito.any(String.class), Mockito.any(String.class), Mockito.any(String.class),
          Mockito.any(String.class), Mockito.any(String.class), Mockito.any(Optional.class),
          Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class),
          Mockito.any(Optional.class), Mockito.any(String.class));
  Mockito
      .doNothing()
      .when(awsSdkClient)
      .deleteAutoScalingGroup(Mockito.any(String.class), Mockito.any(boolean.class));
  Mockito
      .doNothing()
      .when(awsSdkClient)
      .deleteLaunchConfiguration(Mockito.any(String.class));
  Mockito.doNothing()
      .when(awsSdkClient)
      .addPermissionsToSecurityGroup(Mockito.any(String.class), Mockito.any(String.class), Mockito.any(String.class),
          Mockito.any(Integer.class), Mockito.any(Integer.class));

  // Local test Zookeeper
  final TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
  LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());
  this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

  // Load configuration
  final URL url = GobblinAWSClusterLauncherTest.class.getClassLoader().getResource(
      GobblinAWSClusterLauncherTest.class.getSimpleName() + ".conf");
  Assert.assertNotNull(url, "Could not find resource " + url);
  this.config = ConfigFactory.parseURL(url)
      .withValue("gobblin.cluster.zk.connection.string",
                 ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString()))
      .resolve();
  this.helixClusterName = this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY);

  final String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
  this.helixManager = HelixManagerFactory
      .getZKHelixManager(this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY),
          TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);

  // Gobblin AWS Cluster Launcher to test
  this.gobblinAwsClusterLauncher = new GobblinAWSClusterLauncher(this.config);
}