com.amazonaws.services.elasticmapreduce.model.ClusterSummary Java Examples

@Test
public void testGetActiveEmrClusterIdNoIdSpecifiedAssertReturnActualClusterId()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = null;
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getActiveEmrClusterByNameAndAccountId(any(), any(), any()))
            .thenReturn(new ClusterSummary().withId(expectedEmrClusterId).withName(emrClusterName));

        assertEquals(expectedEmrClusterId, emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null));

        verify(mockEmrDao).getActiveEmrClusterByNameAndAccountId(eq(emrClusterName), any(), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}
 

protected List<String> getActiveTaggedClusters() throws Exception{
	AmazonElasticMapReduceClient emrClient = new AmazonElasticMapReduceClient();
	List<String> waitingClusters = new ArrayList<String>();
	ListClustersResult clusterResult = emrClient.listClusters(new ListClustersRequest().withClusterStates(ClusterState.WAITING));
	
	DescribeClusterRequest specifcTagDescribe = new DescribeClusterRequest();
	specifcTagDescribe.putCustomQueryParameter("Cluster.Tags",null);
	 for( ClusterSummary cluster : clusterResult.getClusters()){
		 	System.out.println("list cluster id "+cluster.getId());
		 	List<Tag> tagList = emrClient.describeCluster(specifcTagDescribe.withClusterId(cluster.getId())).getCluster().getTags();
		 	for(Tag tag:tagList){
		 		if(tag.getKey().equals(props.getProperty("edba.cluster.tag.key"))){
		 			waitingClusters.add(cluster.getId());
		 		}
		 	}
		 	
	}
	return waitingClusters;
	
}
 

@Test
public void testGetActiveEmrClusterByNameWhenClusterNameIsBlank()
{
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto =
        new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT,
            AWS_REGION_NAME_US_EAST_1);

    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByNameAndAccountId(BLANK_TEXT, null, awsParamsDto);

    // Verify the external calls.
    verifyNoMoreInteractionsHelper();

    // Validate the results.
    assertNull(result);
}
 

@Test
public void terminateEmrCluster()
{
    String clusterName = "clusterName";
    boolean overrideTerminationProtection = false;
    String clusterId = "clusterId";

    ListClustersResult listClustersResult = new ListClustersResult();
    listClustersResult.setClusters(new ArrayList<>());
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId(clusterId);
    clusterSummary.setName(clusterName);
    listClustersResult.getClusters().add(clusterSummary);
    when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult);

    emrDao.terminateEmrCluster(clusterId, overrideTerminationProtection, getAwsParamsDto());

    // Assert that terminateEmrCluster was called with these parameters ONCE
    verify(mockEmrOperations).terminateEmrCluster(any(), eq(clusterId), eq(overrideTerminationProtection));
}
 

/**
 * Calls ListClusters and DescribeCluster on the AWS EMR Client returning all clusters that match the supplied
 * predicate and attempting to push down certain predicates (namely queries for specific cluster) to EC2.
 *
 * @See TableProvider
 */
@Override
public void readWithConstraint(BlockSpiller spiller, ReadRecordsRequest recordsRequest, QueryStatusChecker queryStatusChecker)
{
    boolean done = false;
    ListClustersRequest request = new ListClustersRequest();

    while (!done) {
        ListClustersResult response = emr.listClusters(request);

        for (ClusterSummary next : response.getClusters()) {
            Cluster cluster = null;
            if (!next.getStatus().getState().toLowerCase().contains("terminated")) {
                DescribeClusterResult clusterResponse = emr.describeCluster(new DescribeClusterRequest().withClusterId(next.getId()));
                cluster = clusterResponse.getCluster();
            }
            clusterToRow(next, cluster, spiller);
        }

        request.setMarker(response.getMarker());

        if (response.getMarker() == null || !queryStatusChecker.isQueryRunning()) {
            done = true;
        }
    }
}
 

@Test
public void testGetActiveEmrClusterByName() throws Exception
{
    // Get the EMR cluster definition object
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);

    // check cluster summary before creation
    ClusterSummary clusterSummary = emrDao
        .getActiveEmrClusterByNameAndAccountId(MockEmrOperationsImpl.MOCK_CLUSTER_NAME, emrClusterDefinition.getAccountId(), emrHelper.getAwsParamsDto());
    assertNull(clusterSummary);

    // Create the cluster
    String clusterId = emrDao.createEmrCluster(MockEmrOperationsImpl.MOCK_CLUSTER_NAME, emrClusterDefinition, emrHelper.getAwsParamsDto());

    // check cluster summary after creation
    clusterSummary = emrDao
        .getActiveEmrClusterByNameAndAccountId(MockEmrOperationsImpl.MOCK_CLUSTER_NAME, emrClusterDefinition.getAccountId(), emrHelper.getAwsParamsDto());
    assertNotNull(clusterSummary);
    assertEquals(clusterId, clusterSummary.getId());
}
 

public List<ClusterSummary> getStackClustersSummary(AmazonElasticMapReduce amazonElasticMapReduce,
        List<String> stackClusterIds, CFTStackInfo cftStackInfo) {
    List<ClusterSummary> stackClustersSummary = new ArrayList<>();
    ListClustersRequest listClustersRequest = new ListClustersRequest();
    //Only get clusters that got created after we setup our stack
    listClustersRequest.setCreatedAfter(cftStackInfo.creationTime());

    ListClustersResult listClustersResult = amazonElasticMapReduce
            .listClusters(listClustersRequest);
    while (true) {
        for (ClusterSummary cluster : listClustersResult.getClusters()) {
            if (stackClusterIds.contains(cluster.getId())) {
                stackClustersSummary.add(cluster);
            }
        }
        if (listClustersResult.getMarker() != null) {
            listClustersRequest.setMarker(listClustersResult.getMarker());
            listClustersResult = amazonElasticMapReduce.listClusters(listClustersRequest);
        }
        else {
            break;
        }
    }
    return stackClustersSummary;
}
 

/**
 * Fetch EMR info.
 *
 * @param temporaryCredentials the temporary credentials
 * @param skipRegions the skip regions
 * @param accountId the accountId
 * @param accountName the account name
 * @return the map
 */
public static Map<String,List<Cluster>> fetchEMRInfo(BasicSessionCredentials temporaryCredentials, String skipRegions,String accountId,String accountName){

	Map<String,List<Cluster>> clusterList = new LinkedHashMap<>();
	String expPrefix = InventoryConstants.ERROR_PREFIX_CODE+accountId + "\",\"Message\": \"Exception in fetching info for resource in specific region\" ,\"type\": \"EMR\" , \"region\":\"" ;
	for(Region region : RegionUtils.getRegions()){
		try{
			if(!skipRegions.contains(region.getName())){
				AmazonElasticMapReduce emrClient = AmazonElasticMapReduceClientBuilder.standard().
				 	withCredentials(new AWSStaticCredentialsProvider(temporaryCredentials)).withRegion(region.getName()).build();
				List<ClusterSummary> clusters = new ArrayList<>();
				String marker = null;
				ListClustersResult clusterResult ;
				do{
					clusterResult = emrClient.listClusters(new ListClustersRequest().withMarker(marker));
					clusters.addAll(clusterResult.getClusters());
					marker = clusterResult.getMarker();
				}while(marker!=null);

				List<Cluster> clustersList = new ArrayList<>();
				clusters.forEach(cluster ->
					{
						DescribeClusterResult descClstrRslt = emrClient.describeCluster(new DescribeClusterRequest().withClusterId(cluster.getId()));
						clustersList.add(descClstrRslt.getCluster());
					});

				if( !clustersList.isEmpty() ){
					log.debug(InventoryConstants.ACCOUNT + accountId +" Type : EMR "+region.getName() + " >> "+clustersList.size());
					clusterList.put(accountId+delimiter+accountName+delimiter+region.getName(),clustersList);
				}
			}
		}catch(Exception e){
			if(region.isServiceSupported(AmazonElasticMapReduce.ENDPOINT_PREFIX)){
				log.warn(expPrefix+ region.getName()+InventoryConstants.ERROR_CAUSE +e.getMessage()+"\"}");
				ErrorManageUtil.uploadError(accountId,region.getName(),"emr",e.getMessage());
			}
		}
	}
	return clusterList;
}
 

/**
 * Maps an EBS Volume into a row in our Apache Arrow response block(s).
 *
 * @param clusterSummary The CluserSummary for the provided Cluster.
 * @param cluster The EMR Cluster to map.
 * @param spiller The BlockSpiller to use when we want to write a matching row to the response.
 * @note The current implementation is rather naive in how it maps fields. It leverages a static
 * list of fields that we'd like to provide and then explicitly filters and converts each field.
 */
private void clusterToRow(ClusterSummary clusterSummary,
        Cluster cluster,
        BlockSpiller spiller)
{
    spiller.writeRows((Block block, int row) -> {
        boolean matched = true;

        matched &= block.offerValue("id", row, clusterSummary.getId());
        matched &= block.offerValue("name", row, clusterSummary.getName());
        matched &= block.offerValue("instance_hours", row, clusterSummary.getNormalizedInstanceHours());
        matched &= block.offerValue("state", row, clusterSummary.getStatus().getState());
        matched &= block.offerValue("state_code", row, clusterSummary.getStatus().getStateChangeReason().getCode());
        matched &= block.offerValue("state_msg", row, clusterSummary.getStatus().getStateChangeReason().getMessage());

        if (cluster != null) {
            matched &= block.offerValue("autoscaling_role", row, cluster.getAutoScalingRole());
            matched &= block.offerValue("custom_ami", row, cluster.getCustomAmiId());
            matched &= block.offerValue("instance_collection_type", row, cluster.getInstanceCollectionType());
            matched &= block.offerValue("log_uri", row, cluster.getLogUri());
            matched &= block.offerValue("master_public_dns", row, cluster.getMasterPublicDnsName());
            matched &= block.offerValue("release_label", row, cluster.getReleaseLabel());
            matched &= block.offerValue("running_ami", row, cluster.getRunningAmiVersion());
            matched &= block.offerValue("scale_down_behavior", row, cluster.getScaleDownBehavior());
            matched &= block.offerValue("service_role", row, cluster.getServiceRole());
            matched &= block.offerValue("service_role", row, cluster.getServiceRole());

            List<String> applications = cluster.getApplications().stream()
                    .map(next -> next.getName() + ":" + next.getVersion()).collect(Collectors.toList());
            matched &= block.offerComplexValue("applications", row, FieldResolver.DEFAULT, applications);

            List<String> tags = cluster.getTags().stream()
                    .map(next -> next.getKey() + ":" + next.getValue()).collect(Collectors.toList());
            matched &= block.offerComplexValue("tags", row, FieldResolver.DEFAULT, tags);
        }

        return matched ? 1 : 0;
    });
}
 

/**
 * Helper method to determine if an Amazon EMR cluster exists
 * 
 * @param client
 *        The {@link AmazonElasticMapReduceClient} with read permissions
 * @param clusterIdentifier
 *        The Amazon EMR cluster to check
 * @return true if the Amazon EMR cluster exists, otherwise false
 */
public static boolean clusterExists(AmazonElasticMapReduce client, String clusterIdentifier) {
	if (clusterIdentifier != null && !clusterIdentifier.isEmpty()) {
		ListClustersResult clustersList = client.listClusters();
		ListIterator<ClusterSummary> iterator = clustersList.getClusters().listIterator();
		ClusterSummary summary;
		for (summary = iterator.next() ; iterator.hasNext();summary = iterator.next()) {
			if (summary.getId().equals(clusterIdentifier)) {
				DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(clusterIdentifier);	
				DescribeClusterResult result = client.describeCluster(describeClusterRequest);	
				if (result != null) {
					Cluster cluster = result.getCluster();
					//check if HBase is installed on this cluster
					if (isHBaseInstalled(client, cluster.getId())) return false;
					String state = cluster.getStatus().getState();
					LOG.info(clusterIdentifier + " is " + state + ". ");
					if (state.equalsIgnoreCase("RUNNING") ||state.equalsIgnoreCase("WAITING"))	{
						LOG.info("The cluster with id " + clusterIdentifier + " exists and is " + state);   
						return true;
					}
				}
			}		
		}					
	}
	LOG.info("The cluster with id " + clusterIdentifier + " does not exist");
	return false;  
}
 

@Override
public String getActiveCluster(String clusterName) {
  ListClustersRequest listClustersRequest = new ListClustersRequest().withClusterStates(CLUSTER_ACTIVE_STATES);
  ListClustersResult result = getEmrClient(emrClusterConfig).listClusters(listClustersRequest);
  LOG.info("Got clusters " + result.getClusters());
  Optional<ClusterSummary> clusterSummary = result.getClusters().stream().filter(cs -> cs.getName().equals(
      clusterName)).findAny();
  if (clusterSummary.isPresent()) {
    return clusterSummary.get().getId();
  }
  return null;
}
 

private ClusterSummary makeClusterSummary(String id)
{
    return new ClusterSummary()
            .withName("name")
            .withId(id)
            .withStatus(new ClusterStatus()
                    .withState("state")
                    .withStateChangeReason(new ClusterStateChangeReason()
                            .withCode("state_code")
                            .withMessage("state_msg")))
            .withNormalizedInstanceHours(100);
}
 

@Override
public ListClustersResult listEmrClusters(AmazonElasticMapReduceClient emrClient, ListClustersRequest listClustersRequest)
{
    List<ClusterSummary> clusterSummaryList = new ArrayList<>();
    for (MockEmrJobFlow cluster : emrClusters.values())
    {
        if (!listClustersRequest.getClusterStates().isEmpty() && listClustersRequest.getClusterStates().contains(cluster.getStatus()))
        {
            ClusterSummary clusterSummary = new ClusterSummary();
            clusterSummary.withId(cluster.getJobFlowId()).withName(cluster.getJobFlowName()).withStatus(new ClusterStatus().withState(cluster.getStatus())
                .withStateChangeReason(new ClusterStateChangeReason().withCode(cluster.getStatusChangeReason().getCode())
                    .withMessage(cluster.getStatusChangeReason().getMessage())).withTimeline(new ClusterTimeline().withCreationDateTime(
                    cluster.getStatusTimeline().getCreationTime() != null ? cluster.getStatusTimeline().getCreationTime().toGregorianCalendar().getTime() :
                        null).withEndDateTime(
                    cluster.getStatusTimeline().getEndTime() != null ? cluster.getStatusTimeline().getEndTime().toGregorianCalendar().getTime() : null)
                    .withReadyDateTime(
                        cluster.getStatusTimeline().getReadyTime() != null ? cluster.getStatusTimeline().getReadyTime().toGregorianCalendar().getTime() :
                            null)));
            clusterSummaryList.add(clusterSummary);
        }
    }
    if (StringUtils.isBlank(listClustersRequest.getMarker()))
    {
        return new ListClustersResult().withClusters(clusterSummaryList).withMarker(MOCK_EMR_MAKER);
    }
    else
    {
        return new ListClustersResult().withClusters(clusterSummaryList);
    }
}
 

@Test
public void addEmrMasterSecurityGroupsThrowWhenNoInstancesFound()
{
    String clusterName = "clusterName";
    List<String> securityGroups = Lists.newArrayList("securityGroup");
    AwsParamsDto awsParams = getAwsParamsDto();

    ListClustersResult listClustersResult = new ListClustersResult();
    listClustersResult.setClusters(new ArrayList<>());
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId("clusterId");
    clusterSummary.setName(clusterName);
    listClustersResult.getClusters().add(clusterSummary);
    when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult);

    when(mockEmrOperations.listClusterInstancesRequest(any(), any())).thenReturn(new ListInstancesResult());

    try
    {
        emrDao.addEmrMasterSecurityGroups(clusterName, securityGroups, awsParams);
        fail();
    }
    catch (Exception e)
    {
        assertEquals(IllegalArgumentException.class, e.getClass());
        assertEquals("No master instances found for the cluster \"" + clusterName + "\".", e.getMessage());
    }
}
 

@Test
public void addEmrMasterSecurityGroupsCallsEc2AddSecurityGroup() throws Exception
{
    String clusterName = "clusterName";
    List<String> securityGroups = Lists.newArrayList("securityGroup");
    AwsParamsDto awsParams = getAwsParamsDto();
    String ec2InstanceId = "ec2InstanceId";

    ListClustersResult listClustersResult = new ListClustersResult();
    listClustersResult.setClusters(new ArrayList<>());
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId("clusterId");
    clusterSummary.setName(clusterName);
    listClustersResult.getClusters().add(clusterSummary);
    when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult);

    ListInstancesResult listInstancesResult = new ListInstancesResult();
    listInstancesResult.setInstances(new ArrayList<>());
    Instance instance = new Instance();
    instance.setEc2InstanceId(ec2InstanceId);
    listInstancesResult.getInstances().add(instance);
    when(mockEmrOperations.listClusterInstancesRequest(any(), any())).thenReturn(listInstancesResult);

    emrDao.addEmrMasterSecurityGroups(clusterName, securityGroups, awsParams);

    verify(mockEc2Dao).addSecurityGroupsToEc2Instance(eq(ec2InstanceId), eq(securityGroups), any());
    verifyNoMoreInteractions(mockEc2Dao);
}
 

/**
 * Fetch EMR info test.
 *
 * @throws Exception the exception
 */
@SuppressWarnings("static-access")
@Test
public void fetchEMRInfoTest() throws Exception {
    
    mockStatic(AmazonElasticMapReduceClientBuilder.class);
    AmazonElasticMapReduce emrClient = PowerMockito.mock(AmazonElasticMapReduce.class);
    AmazonElasticMapReduceClientBuilder amazonElasticFileSystemClientBuilder = PowerMockito.mock(AmazonElasticMapReduceClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonElasticFileSystemClientBuilder.standard()).thenReturn(amazonElasticFileSystemClientBuilder);
    when(amazonElasticFileSystemClientBuilder.withCredentials(anyObject())).thenReturn(amazonElasticFileSystemClientBuilder);
    when(amazonElasticFileSystemClientBuilder.withRegion(anyString())).thenReturn(amazonElasticFileSystemClientBuilder);
    when(amazonElasticFileSystemClientBuilder.build()).thenReturn(emrClient);
    
    ListClustersResult listClustersResult = new ListClustersResult();
    List<ClusterSummary> clusters = new ArrayList<>();
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId("id");
    clusters.add(clusterSummary);
    listClustersResult.setClusters(clusters);
    when(emrClient.listClusters(anyObject())).thenReturn(listClustersResult);
    
    DescribeClusterResult describeClusterResult = new DescribeClusterResult();
    describeClusterResult.setCluster(new Cluster());
    when(emrClient.describeCluster(anyObject())).thenReturn(describeClusterResult);
    assertThat(inventoryUtil.fetchEMRInfo(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(1));
    
}
 

@Test
public void testEmrCreateClusterAwsSpecificStepsImplDryRun()
{
    // Create an AWS params DTO
    AwsParamsDto awsParamsDto = new AwsParamsDto();

    // Create an EMR cluster definition object
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    emrClusterDefinition.setAccountId(AWS_ACCOUNT_ID);
    emrClusterDefinition.setInstanceDefinitions(new InstanceDefinitions());

    // Create an EMR cluster create request
    EmrClusterCreateRequest emrClusterCreateRequest =
        new EmrClusterCreateRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, DRY_RUN, emrClusterDefinition);
    emrClusterCreateRequest.setEmrClusterDefinitionOverride(emrClusterDefinition);

    // Create an EMR cluster alternate key DTO
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = new EmrClusterAlternateKeyDto(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME);

    // Create a cluster summary object
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId(EMR_CLUSTER_ID);

    // Mock the external calls.
    when(emrHelper.getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId())).thenReturn(awsParamsDto);
    when(emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions())).thenReturn(false);

    // Call the method under test.
    emrHelperServiceImpl.emrCreateClusterAwsSpecificSteps(emrClusterCreateRequest, emrClusterDefinition, emrClusterAlternateKeyDto);

    // Verify the external calls.
    verify(emrHelper).getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId());
    verify(emrHelper).isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions());
    verify(emrPricingHelper).updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto);
    verifyNoMoreInteractionsHelper();
}
 

/**
 * Gets details of an existing EMR Cluster.
 *
 * @param emrClusterAlternateKeyDto the EMR cluster alternate key
 * @param emrClusterId the cluster id of the cluster to get details
 * @param emrStepId the step id of the step to get details
 * @param verbose parameter for whether to return detailed information
 * @param accountId the optional AWS account that EMR cluster is running in
 * @param retrieveInstanceFleets parameter for whether to retrieve instance fleets
 *
 * @return the EMR Cluster object with details.
 */
protected EmrCluster getClusterImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, String emrClusterId, String emrStepId, boolean verbose,
    String accountId, Boolean retrieveInstanceFleets)
{
    AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAccountId(accountId);

    // Perform the request validation.
    validateEmrClusterKey(emrClusterAlternateKeyDto);

    // Get the EMR cluster definition and ensure it exists.
    EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(
        new EmrClusterDefinitionKey(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName()));

    EmrCluster emrCluster = createEmrClusterFromRequest(null, emrClusterDefinitionEntity.getNamespace().getCode(), emrClusterDefinitionEntity.getName(),
        emrClusterAlternateKeyDto.getEmrClusterName(), accountId, null, null, null, null);
    String clusterName = emrHelper.buildEmrClusterName(emrClusterDefinitionEntity.getNamespace().getCode(), emrClusterDefinitionEntity.getName(),
        emrClusterAlternateKeyDto.getEmrClusterName());
    try
    {
        // Get Cluster status if clusterId is specified
        if (StringUtils.isNotBlank(emrClusterId))
        {
            Cluster cluster = emrDao.getEmrClusterById(emrClusterId.trim(), awsParamsDto);

            // Validate that, Cluster exists
            Assert.notNull(cluster, "An EMR cluster must exists with the cluster ID \"" + emrClusterId + "\".");

            // Validate that, Cluster name match as specified
            Assert.isTrue(clusterName.equalsIgnoreCase(cluster.getName()),
                "Cluster name of specified cluster id \"" + emrClusterId + "\" must match the name specified.");
            emrCluster.setId(cluster.getId());
            setEmrClusterStatus(emrCluster, cluster.getStatus());
        }
        else
        {
            ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByNameAndAccountId(clusterName, accountId, awsParamsDto);

            // Validate that, Cluster exists with the name
            Assert.notNull(clusterSummary, "An EMR cluster must exists with the name \"" + clusterName + "\".");

            emrCluster.setId(clusterSummary.getId());
            setEmrClusterStatus(emrCluster, clusterSummary.getStatus());
        }

        // Get active step details
        if (emrHelper.isActiveEmrState(emrCluster.getStatus()))
        {
            StepSummary stepSummary = emrDao.getClusterActiveStep(emrCluster.getId(), awsParamsDto);
            if (stepSummary != null)
            {
                EmrStep activeStep;

                // If verbose get active step details
                if (verbose)
                {
                    activeStep = buildEmrStepFromAwsStep(stepSummary, true);
                }
                else
                {
                    activeStep = buildEmrStepFromAwsStepSummary(stepSummary);
                }
                emrCluster.setActiveStep(activeStep);
            }
        }

        // Get requested step details
        if (StringUtils.isNotBlank(emrStepId))
        {
            Step step = emrDao.getClusterStep(emrCluster.getId(), emrStepId.trim(), awsParamsDto);

            emrCluster.setStep(buildEmrStepFromAwsStep(step, verbose));
        }

        // Get instance fleet if true
        if (BooleanUtils.isTrue(retrieveInstanceFleets))
        {
            ListInstanceFleetsResult listInstanceFleetsResult = emrDao.getListInstanceFleetsResult(emrCluster.getId(), awsParamsDto);
            emrCluster.setInstanceFleets(emrHelper.buildEmrClusterInstanceFleetFromAwsResult(listInstanceFleetsResult));
        }
    }
    catch (AmazonServiceException ex)
    {
        awsServiceHelper.handleAmazonException(ex, "An Amazon exception occurred while getting EMR cluster details with name \"" + clusterName + "\".");
    }

    return emrCluster;
}
 

@Test
public void getActiveEmrClusterByNameAssertUsesListMarker()
{
    String clusterName = "clusterName";
    String expectedClusterId = "clusterId";

    when(mockEmrOperations.listEmrClusters(any(), any())).then(new Answer<ListClustersResult>()
    {
        @Override
        public ListClustersResult answer(InvocationOnMock invocation)
        {
            ListClustersRequest listClustersRequest = invocation.getArgument(1);
            String marker = listClustersRequest.getMarker();

            ListClustersResult listClustersResult = new ListClustersResult();
            listClustersResult.setClusters(new ArrayList<>());

            /*
             * When no marker is given, this is the request for the first page.
             * Return a known marker. The expectation is that the next call to this method should have a request with this expected marker.
             */
            if (marker == null)
            {
                listClustersResult.setMarker("pagination_marker");
            }
            /*
             * When a marker is given, this is expected to be the subsequent call.
             */
            else
            {
                // Assert that the correct marker is passed in
                assertEquals("pagination_marker", marker);

                ClusterSummary clusterSummary = new ClusterSummary();
                clusterSummary.setId(expectedClusterId);
                clusterSummary.setName(clusterName);
                listClustersResult.getClusters().add(clusterSummary);
            }
            return listClustersResult;
        }
    });

    ClusterSummary result = emrDao.getActiveEmrClusterByNameAndAccountId(clusterName, null, getAwsParamsDto());
    assertNotNull(result);
    assertEquals(expectedClusterId, result.getId());
}
 

@Test
public void testGetActiveEmrClusterByName()
{
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto =
        new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT,
            AWS_REGION_NAME_US_EAST_1);

    // Create a mock AmazonElasticMapReduceClient.
    AmazonElasticMapReduceClient amazonElasticMapReduceClient = mock(AmazonElasticMapReduceClient.class);

    // Create a list cluster request.
    ListClustersRequest listClustersRequest = new ListClustersRequest().withClusterStates(EMR_VALID_STATE);

    // Create a list cluster result with a non-matching cluster and a marker.
    ListClustersResult listClusterResultWithMarker =
        new ListClustersResult().withClusters(new ClusterSummary().withName(INVALID_VALUE).withId(EMR_CLUSTER_ID)).withMarker(MARKER);

    // Create a list cluster request with marker.
    ListClustersRequest listClustersRequestWithMarker = new ListClustersRequest().withClusterStates(EMR_VALID_STATE).withMarker(MARKER);

    // Create a cluster summary.
    ClusterSummary clusterSummary = new ClusterSummary().withName(EMR_CLUSTER_NAME).withId(EMR_CLUSTER_ID);

    // Create a list cluster result with the matching cluster.
    ListClustersResult listClusterResult = new ListClustersResult().withClusters(clusterSummary);

    // Mock the external calls.
    when(configurationHelper.getProperty(ConfigurationValue.EMR_VALID_STATES)).thenReturn(EMR_VALID_STATE);
    when(configurationHelper.getProperty(ConfigurationValue.FIELD_DATA_DELIMITER))
        .thenReturn((String) ConfigurationValue.FIELD_DATA_DELIMITER.getDefaultValue());
    when(awsClientFactory.getEmrClient(awsParamsDto)).thenReturn(amazonElasticMapReduceClient);
    when(emrOperations.listEmrClusters(amazonElasticMapReduceClient, listClustersRequest)).thenReturn(listClusterResultWithMarker);
    when(emrOperations.listEmrClusters(amazonElasticMapReduceClient, listClustersRequestWithMarker)).thenReturn(listClusterResult);

    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, null, awsParamsDto);

    // Verify the external calls.
    verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
    verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
    verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
    verify(emrOperations, times(2)).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    verifyNoMoreInteractionsHelper();

    // Validate the results.
    assertEquals(clusterSummary, result);
}
 

private void testGetActiveEmrClusterByNameWithTimestamps(EmrClusterCacheTimestamps emrClusterCacheTimestamps, Date createdAfter)
{
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto =
        new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT,
            AWS_REGION_NAME_US_EAST_1);

    // Create a mock AmazonElasticMapReduceClient.
    AmazonElasticMapReduceClient amazonElasticMapReduceClient = mock(AmazonElasticMapReduceClient.class);

    // Create a list cluster request.
    ListClustersRequest listClustersRequest = new ListClustersRequest().withClusterStates(EMR_VALID_STATE).withCreatedAfter(createdAfter);

    // Create a list cluster result with a non-matching cluster and a marker.
    ListClustersResult listClusterResultWithMarker =
        new ListClustersResult().withClusters(new ClusterSummary().withName(INVALID_VALUE).withId(EMR_CLUSTER_ID)).withMarker(MARKER);

    // Create a list cluster request with marker.
    ListClustersRequest listClustersRequestWithMarker =
        new ListClustersRequest().withClusterStates(EMR_VALID_STATE).withMarker(MARKER).withCreatedAfter(createdAfter);

    // Create a cluster summary.
    ClusterSummary clusterSummary = new ClusterSummary().withName(EMR_CLUSTER_NAME).withId(EMR_CLUSTER_ID);

    // Create a list cluster result with the matching cluster.
    ListClustersResult listClusterResult = new ListClustersResult().withClusters(clusterSummary);


    // Mock the external calls.
    when(emrClusterCacheTimestampsMap.get(EMR_CLUSTER_CACHE_MAP_DEFAULT_AWS_ACCOUNT_ID_KEY)).thenReturn(emrClusterCacheTimestamps);
    when(configurationHelper.getProperty(ConfigurationValue.EMR_VALID_STATES)).thenReturn(EMR_VALID_STATE);
    when(configurationHelper.getProperty(ConfigurationValue.FIELD_DATA_DELIMITER))
        .thenReturn((String) ConfigurationValue.FIELD_DATA_DELIMITER.getDefaultValue());
    when(awsClientFactory.getEmrClient(awsParamsDto)).thenReturn(amazonElasticMapReduceClient);
    when(emrOperations.listEmrClusters(amazonElasticMapReduceClient, listClustersRequest)).thenReturn(listClusterResultWithMarker);
    when(emrOperations.listEmrClusters(amazonElasticMapReduceClient, listClustersRequestWithMarker)).thenReturn(listClusterResult);

    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, null, awsParamsDto);

    // Verify the external calls.
    verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
    verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
    verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
    verify(emrOperations, times(2)).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    verifyNoMoreInteractionsHelper();

    // Validate the results.
    assertEquals(clusterSummary, result);
}
 

private void getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(Cluster cluster, String accountId)
{
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto =
        new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT,
            AWS_REGION_NAME_US_EAST_1);

    // Create a mock AmazonElasticMapReduceClient.
    AmazonElasticMapReduceClient amazonElasticMapReduceClient = mock(AmazonElasticMapReduceClient.class);

    // Create a cluster summary.
    ClusterSummary clusterSummary =
        new ClusterSummary().withName(EMR_CLUSTER_NAME).withId(EMR_CLUSTER_ID).withStatus(cluster == null ? null : cluster.getStatus());

    // Create a list cluster result with the matching cluster.
    ListClustersResult listClusterResult = new ListClustersResult().withClusters(clusterSummary);

    // Create a describe cluster result.
    DescribeClusterResult describeClusterResult = new DescribeClusterResult().withCluster(cluster);

    // Create a describe cluster request.
    DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(EMR_CLUSTER_ID);

    // Build the EMR cluster cache key
    EmrClusterCacheKey emrClusterCacheKey = new EmrClusterCacheKey(EMR_CLUSTER_NAME.toUpperCase(), accountId);

    // Build the EMR cluster cache
    Map<EmrClusterCacheKey, String> emrClusterCache = new ConcurrentHashMap<>();
    emrClusterCache.put(emrClusterCacheKey, EMR_CLUSTER_ID);

    // Mock the external calls.
    if (accountId == null)
    {
        when(emrClusterCacheMap.get(EMR_CLUSTER_CACHE_MAP_DEFAULT_AWS_ACCOUNT_ID_KEY)).thenReturn(emrClusterCache);
    }
    else
    {
        when(emrClusterCacheMap.get(accountId)).thenReturn(emrClusterCache);
    }

    when(emrOperations.describeClusterRequest(eq(amazonElasticMapReduceClient), any(DescribeClusterRequest.class))).thenReturn(describeClusterResult);
    when(configurationHelper.getProperty(ConfigurationValue.EMR_VALID_STATES)).thenReturn(ConfigurationValue.EMR_VALID_STATES.getDefaultValue().toString());
    when(configurationHelper.getProperty(ConfigurationValue.FIELD_DATA_DELIMITER))
        .thenReturn((String) ConfigurationValue.FIELD_DATA_DELIMITER.getDefaultValue());
    when(awsClientFactory.getEmrClient(awsParamsDto)).thenReturn(amazonElasticMapReduceClient);
    when(emrOperations.listEmrClusters(any(AmazonElasticMapReduceClient.class), any(ListClustersRequest.class))).thenReturn(listClusterResult);

    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, accountId, awsParamsDto);

    // Verify the external calls.
    verify(emrOperations).describeClusterRequest(eq(amazonElasticMapReduceClient), eq(describeClusterRequest));

    if (cluster == null)
    {
        verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
        verify(emrOperations).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    }
    else if (cluster.getStatus().getState().equals(EMR_INVALID_STATE))
    {
        verify(configurationHelper, times(2)).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper, times(2)).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
        verify(emrOperations).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    }
    else
    {
        verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory).getEmrClient(awsParamsDto);
    }

    verifyNoMoreInteractionsHelper();

    // Validate the results.
    assertEquals(clusterSummary, result);
}
 

@Test
public void testEmrCreateClusterAwsSpecificStepsImpl()
{
    // Create an AWS params DTO
    AwsParamsDto awsParamsDto = new AwsParamsDto();

    // Create an EMR cluster definition object
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    emrClusterDefinition.setAccountId(AWS_ACCOUNT_ID);
    emrClusterDefinition.setInstanceDefinitions(new InstanceDefinitions());

    // Create an EMR cluster create request
    EmrClusterCreateRequest emrClusterCreateRequest =
        new EmrClusterCreateRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, NO_DRY_RUN, emrClusterDefinition);
    emrClusterCreateRequest.setEmrClusterDefinitionOverride(emrClusterDefinition);

    // Create an EMR cluster alternate key DTO
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = new EmrClusterAlternateKeyDto(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME);

    // Create a cluster summary object
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId(EMR_CLUSTER_ID);
    clusterSummary.setStatus(new ClusterStatus().withState(EMR_CLUSTER_STATUS));

    // Mock the external calls.
    when(emrHelper.getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId())).thenReturn(awsParamsDto);
    when(emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions())).thenReturn(false);
    when(emrHelper.buildEmrClusterName(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(),
        emrClusterAlternateKeyDto.getEmrClusterName())).thenReturn(EMR_CLUSTER_NAME);
    when(emrDao.getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, emrClusterDefinition.getAccountId(), awsParamsDto)).thenReturn(clusterSummary);

    // Call the method under test.
    emrHelperServiceImpl.emrCreateClusterAwsSpecificSteps(emrClusterCreateRequest, emrClusterDefinition, emrClusterAlternateKeyDto);

    // Verify the external calls.
    verify(emrHelper).getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId());
    verify(emrHelper).isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions());
    verify(emrPricingHelper).updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto);
    verify(emrHelper).buildEmrClusterName(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(),
        emrClusterAlternateKeyDto.getEmrClusterName());
    verify(emrDao).getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, emrClusterDefinition.getAccountId(), awsParamsDto);
    verifyNoMoreInteractionsHelper();
}
 

/**
 * The implementation of the create cluster AWS specific steps.  These steps are run outside of any transaction.
 *
 * @param request the EMR cluster create request
 * @param emrClusterDefinition the EMR cluster definition object
 * @param emrClusterAlternateKeyDto the EMR cluster alternate key data transfer object
 *
 * @return the EMR cluster create data transfer object
 */
EmrClusterCreateDto emrCreateClusterAwsSpecificStepsImpl(EmrClusterCreateRequest request, EmrClusterDefinition emrClusterDefinition,
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto)
{
    String accountId = emrClusterDefinition.getAccountId();

    AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAccountId(accountId);

    // If instance group definitions are specified, find best price and update definition.
    if (!emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions()))
    {
        emrPricingHelper.updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto);
    }

    // The cluster ID record.
    String clusterId = null;

    // Is EMR cluster already existing.
    Boolean emrClusterAlreadyExists = null;

    // Is EMR cluster created.
    Boolean emrClusterCreated = null;

    // EMR cluster status string.
    String emrClusterStatus = null;

    // If the dryRun flag is null or false. This is the default option if no flag is given.
    if (!Boolean.TRUE.equals(request.isDryRun()))
    {
        /*
         * Create the cluster only if the cluster does not already exist.
         * If the cluster is created, record the newly created cluster ID.
         * If the cluster already exists, record the existing cluster ID.
         * If there is any error while attempting to check for existing cluster or create a new one, handle the exception to throw appropriate exception.
         */
        String clusterName = emrHelper
            .buildEmrClusterName(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(),
                emrClusterAlternateKeyDto.getEmrClusterName());
        try
        {
            // Synchronizing this block of code to prevent duplicate cluster creation.
            synchronized (this)
            {
                LOGGER.info("Entering synchronized block.");

                // Try to get an active EMR cluster by its name.
                ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByNameAndAccountId(clusterName, accountId, awsParamsDto);

                // If cluster does not already exist.
                if (clusterSummary == null)
                {
                    clusterId = emrDao.createEmrCluster(clusterName, emrClusterDefinition, awsParamsDto);
                    emrClusterCreated = true;
                    emrClusterStatus = emrDao.getEmrClusterStatusById(clusterId, awsParamsDto);

                }
                // If the cluster already exists.
                else
                {
                    clusterId = clusterSummary.getId();
                    emrClusterCreated = false;
                    emrClusterAlreadyExists = true;

                    // If the cluster already exists use the status from the get active EMR cluster by name and account id method call.
                    emrClusterStatus = clusterSummary.getStatus().getState();
                }

                LOGGER.info("Exiting synchronized block.");
            }
        }
        catch (AmazonServiceException ex)
        {
            awsServiceHelper.handleAmazonException(ex, "An Amazon exception occurred while creating EMR cluster with name \"" + clusterName + "\".");
        }
    }
    // If the dryRun flag is true and not null
    else
    {
        emrClusterCreated = false;
    }

    return new EmrClusterCreateDto(clusterId, emrClusterAlreadyExists, emrClusterCreated, emrClusterStatus);
}
 

/**
 * Get an Active EMR cluster by the cluster name and account id. Cluster only in following states are returned: ClusterState.BOOTSTRAPPING,
 * ClusterState.RUNNING, ClusterState.STARTING, ClusterState.WAITING
 *
 * @param awsParams AWS related parameters for access/secret keys and proxy details.
 * @param clusterName the cluster name value.
 * @param accountId the account id in which the cluster resides.
 *
 * @return the ClusterSummary object.
 */
public ClusterSummary getActiveEmrClusterByNameAndAccountId(String clusterName, String accountId, AwsParamsDto awsParams);