com.amazonaws.services.elasticmapreduce.model.Cluster Java Examples

/**
 * Calls ListClusters and DescribeCluster on the AWS EMR Client returning all clusters that match the supplied
 * predicate and attempting to push down certain predicates (namely queries for specific cluster) to EC2.
 *
 * @See TableProvider
 */
@Override
public void readWithConstraint(BlockSpiller spiller, ReadRecordsRequest recordsRequest, QueryStatusChecker queryStatusChecker)
{
    boolean done = false;
    ListClustersRequest request = new ListClustersRequest();

    while (!done) {
        ListClustersResult response = emr.listClusters(request);

        for (ClusterSummary next : response.getClusters()) {
            Cluster cluster = null;
            if (!next.getStatus().getState().toLowerCase().contains("terminated")) {
                DescribeClusterResult clusterResponse = emr.describeCluster(new DescribeClusterRequest().withClusterId(next.getId()));
                cluster = clusterResponse.getCluster();
            }
            clusterToRow(next, cluster, spiller);
        }

        request.setMarker(response.getMarker());

        if (response.getMarker() == null || !queryStatusChecker.isQueryRunning()) {
            done = true;
        }
    }
}
 

private Cluster makeCluster(String id)
{
    return new Cluster()
            .withId(id)
            .withName("name")
            .withAutoScalingRole("autoscaling_role")
            .withCustomAmiId("custom_ami")
            .withInstanceCollectionType("instance_collection_type")
            .withLogUri("log_uri")
            .withMasterPublicDnsName("master_public_dns")
            .withReleaseLabel("release_label")
            .withRunningAmiVersion("running_ami")
            .withScaleDownBehavior("scale_down_behavior")
            .withServiceRole("service_role")
            .withApplications(new Application().withName("name").withVersion("version"))
            .withTags(new Tag("key", "value"));
}
 

@Test
public void testGetClusterStatus() {
  Properties properties = new Properties();
  EmrClusterJob emrClusterJob = new EmrClusterJob();
  EmrClusterJob.Client client = Mockito.spy(emrClusterJob.getClient(properties));
  AmazonElasticMapReduce emr = Mockito.mock(AmazonElasticMapReduce.class);
  Mockito.doReturn(emr).when(client).getEmrClient(Mockito.any(EmrClusterConfig.class));
  DescribeClusterResult result = Mockito.mock(DescribeClusterResult.class);
  Mockito.doReturn(result).when(emr).describeCluster(Mockito.any(DescribeClusterRequest
      .class));
  Cluster cluster = Mockito.mock(Cluster.class);
  Mockito.doReturn(cluster).when(result).getCluster();
  Mockito.doReturn(Mockito.mock(ClusterStatus.class)).when(cluster).getStatus();
  client.getClusterStatus("foo");
  Mockito.verify(emr, Mockito.times(1)).describeCluster(Mockito.any(DescribeClusterRequest
      .class));
  Mockito.verify(client, Mockito.times(1)).getEmrClient(Mockito.any(EmrClusterConfig.class));
}
 

/**
 * Fetch redshift info test.
 *
 * @throws Exception the exception
 */
@SuppressWarnings("static-access")
@Test
public void fetchRedshiftInfoTest() throws Exception {
    
    mockStatic(AmazonRedshiftClientBuilder.class);
    AmazonRedshift redshiftClient = PowerMockito.mock(AmazonRedshift.class);
    AmazonRedshiftClientBuilder amazonRedshiftClientBuilder = PowerMockito.mock(AmazonRedshiftClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonRedshiftClientBuilder.standard()).thenReturn(amazonRedshiftClientBuilder);
    when(amazonRedshiftClientBuilder.withCredentials(anyObject())).thenReturn(amazonRedshiftClientBuilder);
    when(amazonRedshiftClientBuilder.withRegion(anyString())).thenReturn(amazonRedshiftClientBuilder);
    when(amazonRedshiftClientBuilder.build()).thenReturn(redshiftClient);
    
    DescribeClustersResult describeClustersResult = new DescribeClustersResult();
    List<com.amazonaws.services.redshift.model.Cluster> redshiftList = new ArrayList<>();
    redshiftList.add(new com.amazonaws.services.redshift.model.Cluster());
    describeClustersResult.setClusters(redshiftList);
    when(redshiftClient.describeClusters(anyObject())).thenReturn(describeClustersResult);
    assertThat(inventoryUtil.fetchRedshiftInfo(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(1));
}
 

@Override
public Cluster getEmrClusterById(String clusterId, AwsParamsDto awsParams)
{
    Cluster cluster = null;
    if (StringUtils.isNotBlank(clusterId))
    {
        DescribeClusterResult describeClusterResult =
            emrOperations.describeClusterRequest(getEmrClient(awsParams), new DescribeClusterRequest().withClusterId(clusterId));
        if (describeClusterResult != null && describeClusterResult.getCluster() != null)
        {
            cluster = describeClusterResult.getCluster();
        }
    }

    return cluster;
}
 

@Test
public void testGetActiveEmrClusterIdAssertReturnActualClusterIdWhenClusterIdSpecifiedAndClusterStateActiveAndNameMatch()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId, emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null));

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId.trim()), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}
 

/**
 * Helper method to determine if an Amazon EMR cluster exists
 * 
 * @param client
 *        The {@link AmazonElasticMapReduceClient} with read permissions
 * @param clusterIdentifier
 *        The Amazon EMR cluster to check
 * @return true if the Amazon EMR cluster exists, otherwise false
 */
public static boolean clusterExists(AmazonElasticMapReduce client, String clusterIdentifier) {
	if (clusterIdentifier != null && !clusterIdentifier.isEmpty()) {
		ListClustersResult clustersList = client.listClusters();
		ListIterator<ClusterSummary> iterator = clustersList.getClusters().listIterator();
		ClusterSummary summary;
		for (summary = iterator.next() ; iterator.hasNext();summary = iterator.next()) {
			if (summary.getId().equals(clusterIdentifier)) {
				DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(clusterIdentifier);	
				DescribeClusterResult result = client.describeCluster(describeClusterRequest);	
				if (result != null) {
					Cluster cluster = result.getCluster();
					//check if HBase is installed on this cluster
					if (isHBaseInstalled(client, cluster.getId())) return false;
					String state = cluster.getStatus().getState();
					LOG.info(clusterIdentifier + " is " + state + ". ");
					if (state.equalsIgnoreCase("RUNNING") ||state.equalsIgnoreCase("WAITING"))	{
						LOG.info("The cluster with id " + clusterIdentifier + " exists and is " + state);   
						return true;
					}
				}
			}		
		}					
	}
	LOG.info("The cluster with id " + clusterIdentifier + " does not exist");
	return false;  
}
 

@Test
public void testGetActiveEmrClusterByNameAndAccountIdWhenClusterNameIsInCacheWithInvalidStateWithNullAccountId()
{
    // Create a cluster with an invalid state.
    Cluster cluster = new Cluster().withStatus(new ClusterStatus().withState(EMR_INVALID_STATE));

    // Test that the EMR Cluster Cache is not used.
    getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(cluster, null);
}
 

@Test
public void testGetActiveEmrClusterByNameAndAccountIdWhenClusterNameIsInCacheWithInvalidState()
{
    // Create a cluster with an invalid state.
    Cluster cluster = new Cluster().withStatus(new ClusterStatus().withState(EMR_INVALID_STATE));

    // Test that the EMR Cluster Cache is not used.
    getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(cluster, AWS_ACCOUNT_ID);
}
 

@Test
public void testGetActiveEmrClusterByNameAndAccountIdWhenClusterNameIsInCacheWithNullAccountId()
{
    // Create a cluster with a valid state.
    Cluster cluster = new Cluster().withStatus(new ClusterStatus().withState("STARTING"));

    // Test the EMR Cluster Cache is used.
    getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(cluster, null);
}
 

@Test
public void testGetActiveEmrClusterByNameAndAccountIdWhenClusterNameIsInCache()
{
    // Create a cluster with a valid state.
    Cluster cluster = new Cluster().withStatus(new ClusterStatus().withState("STARTING"));

    // Test the EMR Cluster Cache is used.
    getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(cluster, AWS_ACCOUNT_ID);
}
 

@Test
public void testGetActiveEmrClusterIdAssertParametersCaseIgnored()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId,
            emrHelper.getActiveEmrClusterId(StringUtils.upperCase(emrClusterId), StringUtils.upperCase(emrClusterName), null));

        verify(mockEmrDao).getEmrClusterById(eq(StringUtils.upperCase(emrClusterId)), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}
 

@Test
public void testGetActiveEmrClusterIdAssertParametersTrimmed()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId,
            emrHelper.getActiveEmrClusterId(StringUtils.wrap(emrClusterId, BLANK_TEXT), StringUtils.wrap(emrClusterName, BLANK_TEXT), null));

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId.trim()), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}
 

@Test
public void testGetActiveEmrClusterIdAssertErrorWhenClusterIdSpecifiedAndClusterStateNotActive()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        ClusterState actualClusterState = ClusterState.TERMINATED;
        when(mockEmrDao.getEmrClusterById(any(), any()))
            .thenReturn(new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(actualClusterState)));

        try
        {
            emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null);
            fail();
        }
        catch (IllegalArgumentException e)
        {
            assertEquals(String.format("The cluster with ID \"%s\" is not active. The cluster state must be in one of [STARTING, BOOTSTRAPPING, RUNNING, " +
                "WAITING]. Current state is \"%s\"", emrClusterId, actualClusterState), e.getMessage());
        }

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}
 

@Test
public void testGetActiveEmrClusterIdAssertReturnActualClusterIdWhenClusterStateActiveAndNameNotSpecified()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = null;
        String expectedEmrClusterId = "expectedEmrClusterId";
        String actualEmrClusterName = "actualEmrClusterName";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(actualEmrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId, emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null));

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}
 

@Test
public void testGetActiveEmrClusterIdAssertErrorWhenClusterIdSpecifiedAndNameMismatch()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";
        String actualEmrClusterName = "actualEmrClusterName";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(actualEmrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        try
        {
            emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null);
            fail();
        }
        catch (IllegalArgumentException e)
        {
            assertEquals(String
                .format("The cluster with ID \"%s\" does not match the expected name \"%s\". The actual name is \"%s\".", expectedEmrClusterId,
                    emrClusterName, actualEmrClusterName), e.getMessage());
        }

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId.trim()), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}
 

@Override
public String getEmrClusterStatusById(String clusterId, AwsParamsDto awsParams)
{
    Cluster cluster = getEmrClusterById(clusterId, awsParams);

    return ((cluster == null) ? null : cluster.getStatus().getState());
}
 

@Test
public void testGetEmrClusterByIdNull()
{
    Cluster cluster = emrDao.getEmrClusterById(null, null);

    assertNull(cluster);
}
 

/**
 * Fetch EMR info test.
 *
 * @throws Exception the exception
 */
@SuppressWarnings("static-access")
@Test
public void fetchEMRInfoTest() throws Exception {
    
    mockStatic(AmazonElasticMapReduceClientBuilder.class);
    AmazonElasticMapReduce emrClient = PowerMockito.mock(AmazonElasticMapReduce.class);
    AmazonElasticMapReduceClientBuilder amazonElasticFileSystemClientBuilder = PowerMockito.mock(AmazonElasticMapReduceClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonElasticFileSystemClientBuilder.standard()).thenReturn(amazonElasticFileSystemClientBuilder);
    when(amazonElasticFileSystemClientBuilder.withCredentials(anyObject())).thenReturn(amazonElasticFileSystemClientBuilder);
    when(amazonElasticFileSystemClientBuilder.withRegion(anyString())).thenReturn(amazonElasticFileSystemClientBuilder);
    when(amazonElasticFileSystemClientBuilder.build()).thenReturn(emrClient);
    
    ListClustersResult listClustersResult = new ListClustersResult();
    List<ClusterSummary> clusters = new ArrayList<>();
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId("id");
    clusters.add(clusterSummary);
    listClustersResult.setClusters(clusters);
    when(emrClient.listClusters(anyObject())).thenReturn(listClustersResult);
    
    DescribeClusterResult describeClusterResult = new DescribeClusterResult();
    describeClusterResult.setCluster(new Cluster());
    when(emrClient.describeCluster(anyObject())).thenReturn(describeClusterResult);
    assertThat(inventoryUtil.fetchEMRInfo(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(1));
    
}
 

/**
 * Fetch EMR info.
 *
 * @param temporaryCredentials the temporary credentials
 * @param skipRegions the skip regions
 * @param accountId the accountId
 * @param accountName the account name
 * @return the map
 */
public static Map<String,List<Cluster>> fetchEMRInfo(BasicSessionCredentials temporaryCredentials, String skipRegions,String accountId,String accountName){

	Map<String,List<Cluster>> clusterList = new LinkedHashMap<>();
	String expPrefix = InventoryConstants.ERROR_PREFIX_CODE+accountId + "\",\"Message\": \"Exception in fetching info for resource in specific region\" ,\"type\": \"EMR\" , \"region\":\"" ;
	for(Region region : RegionUtils.getRegions()){
		try{
			if(!skipRegions.contains(region.getName())){
				AmazonElasticMapReduce emrClient = AmazonElasticMapReduceClientBuilder.standard().
				 	withCredentials(new AWSStaticCredentialsProvider(temporaryCredentials)).withRegion(region.getName()).build();
				List<ClusterSummary> clusters = new ArrayList<>();
				String marker = null;
				ListClustersResult clusterResult ;
				do{
					clusterResult = emrClient.listClusters(new ListClustersRequest().withMarker(marker));
					clusters.addAll(clusterResult.getClusters());
					marker = clusterResult.getMarker();
				}while(marker!=null);

				List<Cluster> clustersList = new ArrayList<>();
				clusters.forEach(cluster ->
					{
						DescribeClusterResult descClstrRslt = emrClient.describeCluster(new DescribeClusterRequest().withClusterId(cluster.getId()));
						clustersList.add(descClstrRslt.getCluster());
					});

				if( !clustersList.isEmpty() ){
					log.debug(InventoryConstants.ACCOUNT + accountId +" Type : EMR "+region.getName() + " >> "+clustersList.size());
					clusterList.put(accountId+delimiter+accountName+delimiter+region.getName(),clustersList);
				}
			}
		}catch(Exception e){
			if(region.isServiceSupported(AmazonElasticMapReduce.ENDPOINT_PREFIX)){
				log.warn(expPrefix+ region.getName()+InventoryConstants.ERROR_CAUSE +e.getMessage()+"\"}");
				ErrorManageUtil.uploadError(accountId,region.getName(),"emr",e.getMessage());
			}
		}
	}
	return clusterList;
}
 

/**
 * Generate emr files.
 *
 * @param fileInofMap the file inof map
 * @throws IOException Signals that an I/O exception has occurred.
 */
public static void generateEmrFiles(Map<String,List<Cluster>> fileInofMap) throws IOException {
	String fieldNames;
	String keys;
	fieldNames ="Id`AutoScalingRole`AutoTerminate`InstanceCollectionType`LogUri`MasterPublicDnsName`Name`NormalizedInstanceHours`ReleaseLabel`RequestedAmiVersion`RunningAmiVersion`ScaleDownBehavior`SecurityConfiguration`ServiceRole`TerminationProtected`VisibleToAllUsers";
	keys ="discoverydate`accountid`accountname`region`id`autoscalingrole`autoterminate`instancecollectiontype`loguri`masterpubdnsname`name`norminstancehours`releaselabel`reqamiversion`runningamiversion`scaledownbehavior`securityconfig`servicerole`terminationprotected`visibletoallusers";
	FileGenerator.generateJson(fileInofMap, fieldNames, "aws-emr.data",keys);
	fieldNames ="Id`tags.key`tags.value";
	keys ="discoverydate`accountid`accountname`region`id`key`value";
	FileGenerator.generateJson(fileInofMap, fieldNames, "aws-emr-tags.data",keys);

}
 

/**
 * Maps an EBS Volume into a row in our Apache Arrow response block(s).
 *
 * @param clusterSummary The CluserSummary for the provided Cluster.
 * @param cluster The EMR Cluster to map.
 * @param spiller The BlockSpiller to use when we want to write a matching row to the response.
 * @note The current implementation is rather naive in how it maps fields. It leverages a static
 * list of fields that we'd like to provide and then explicitly filters and converts each field.
 */
private void clusterToRow(ClusterSummary clusterSummary,
        Cluster cluster,
        BlockSpiller spiller)
{
    spiller.writeRows((Block block, int row) -> {
        boolean matched = true;

        matched &= block.offerValue("id", row, clusterSummary.getId());
        matched &= block.offerValue("name", row, clusterSummary.getName());
        matched &= block.offerValue("instance_hours", row, clusterSummary.getNormalizedInstanceHours());
        matched &= block.offerValue("state", row, clusterSummary.getStatus().getState());
        matched &= block.offerValue("state_code", row, clusterSummary.getStatus().getStateChangeReason().getCode());
        matched &= block.offerValue("state_msg", row, clusterSummary.getStatus().getStateChangeReason().getMessage());

        if (cluster != null) {
            matched &= block.offerValue("autoscaling_role", row, cluster.getAutoScalingRole());
            matched &= block.offerValue("custom_ami", row, cluster.getCustomAmiId());
            matched &= block.offerValue("instance_collection_type", row, cluster.getInstanceCollectionType());
            matched &= block.offerValue("log_uri", row, cluster.getLogUri());
            matched &= block.offerValue("master_public_dns", row, cluster.getMasterPublicDnsName());
            matched &= block.offerValue("release_label", row, cluster.getReleaseLabel());
            matched &= block.offerValue("running_ami", row, cluster.getRunningAmiVersion());
            matched &= block.offerValue("scale_down_behavior", row, cluster.getScaleDownBehavior());
            matched &= block.offerValue("service_role", row, cluster.getServiceRole());
            matched &= block.offerValue("service_role", row, cluster.getServiceRole());

            List<String> applications = cluster.getApplications().stream()
                    .map(next -> next.getName() + ":" + next.getVersion()).collect(Collectors.toList());
            matched &= block.offerComplexValue("applications", row, FieldResolver.DEFAULT, applications);

            List<String> tags = cluster.getTags().stream()
                    .map(next -> next.getKey() + ":" + next.getValue()).collect(Collectors.toList());
            matched &= block.offerComplexValue("tags", row, FieldResolver.DEFAULT, tags);
        }

        return matched ? 1 : 0;
    });
}
 

/**
 * Gets details of an existing EMR Cluster.
 *
 * @param emrClusterAlternateKeyDto the EMR cluster alternate key
 * @param emrClusterId the cluster id of the cluster to get details
 * @param emrStepId the step id of the step to get details
 * @param verbose parameter for whether to return detailed information
 * @param accountId the optional AWS account that EMR cluster is running in
 * @param retrieveInstanceFleets parameter for whether to retrieve instance fleets
 *
 * @return the EMR Cluster object with details.
 */
protected EmrCluster getClusterImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, String emrClusterId, String emrStepId, boolean verbose,
    String accountId, Boolean retrieveInstanceFleets)
{
    AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAccountId(accountId);

    // Perform the request validation.
    validateEmrClusterKey(emrClusterAlternateKeyDto);

    // Get the EMR cluster definition and ensure it exists.
    EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(
        new EmrClusterDefinitionKey(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName()));

    EmrCluster emrCluster = createEmrClusterFromRequest(null, emrClusterDefinitionEntity.getNamespace().getCode(), emrClusterDefinitionEntity.getName(),
        emrClusterAlternateKeyDto.getEmrClusterName(), accountId, null, null, null, null);
    String clusterName = emrHelper.buildEmrClusterName(emrClusterDefinitionEntity.getNamespace().getCode(), emrClusterDefinitionEntity.getName(),
        emrClusterAlternateKeyDto.getEmrClusterName());
    try
    {
        // Get Cluster status if clusterId is specified
        if (StringUtils.isNotBlank(emrClusterId))
        {
            Cluster cluster = emrDao.getEmrClusterById(emrClusterId.trim(), awsParamsDto);

            // Validate that, Cluster exists
            Assert.notNull(cluster, "An EMR cluster must exists with the cluster ID \"" + emrClusterId + "\".");

            // Validate that, Cluster name match as specified
            Assert.isTrue(clusterName.equalsIgnoreCase(cluster.getName()),
                "Cluster name of specified cluster id \"" + emrClusterId + "\" must match the name specified.");
            emrCluster.setId(cluster.getId());
            setEmrClusterStatus(emrCluster, cluster.getStatus());
        }
        else
        {
            ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByNameAndAccountId(clusterName, accountId, awsParamsDto);

            // Validate that, Cluster exists with the name
            Assert.notNull(clusterSummary, "An EMR cluster must exists with the name \"" + clusterName + "\".");

            emrCluster.setId(clusterSummary.getId());
            setEmrClusterStatus(emrCluster, clusterSummary.getStatus());
        }

        // Get active step details
        if (emrHelper.isActiveEmrState(emrCluster.getStatus()))
        {
            StepSummary stepSummary = emrDao.getClusterActiveStep(emrCluster.getId(), awsParamsDto);
            if (stepSummary != null)
            {
                EmrStep activeStep;

                // If verbose get active step details
                if (verbose)
                {
                    activeStep = buildEmrStepFromAwsStep(stepSummary, true);
                }
                else
                {
                    activeStep = buildEmrStepFromAwsStepSummary(stepSummary);
                }
                emrCluster.setActiveStep(activeStep);
            }
        }

        // Get requested step details
        if (StringUtils.isNotBlank(emrStepId))
        {
            Step step = emrDao.getClusterStep(emrCluster.getId(), emrStepId.trim(), awsParamsDto);

            emrCluster.setStep(buildEmrStepFromAwsStep(step, verbose));
        }

        // Get instance fleet if true
        if (BooleanUtils.isTrue(retrieveInstanceFleets))
        {
            ListInstanceFleetsResult listInstanceFleetsResult = emrDao.getListInstanceFleetsResult(emrCluster.getId(), awsParamsDto);
            emrCluster.setInstanceFleets(emrHelper.buildEmrClusterInstanceFleetFromAwsResult(listInstanceFleetsResult));
        }
    }
    catch (AmazonServiceException ex)
    {
        awsServiceHelper.handleAmazonException(ex, "An Amazon exception occurred while getting EMR cluster details with name \"" + clusterName + "\".");
    }

    return emrCluster;
}
 

private void getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(Cluster cluster, String accountId)
{
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto =
        new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT,
            AWS_REGION_NAME_US_EAST_1);

    // Create a mock AmazonElasticMapReduceClient.
    AmazonElasticMapReduceClient amazonElasticMapReduceClient = mock(AmazonElasticMapReduceClient.class);

    // Create a cluster summary.
    ClusterSummary clusterSummary =
        new ClusterSummary().withName(EMR_CLUSTER_NAME).withId(EMR_CLUSTER_ID).withStatus(cluster == null ? null : cluster.getStatus());

    // Create a list cluster result with the matching cluster.
    ListClustersResult listClusterResult = new ListClustersResult().withClusters(clusterSummary);

    // Create a describe cluster result.
    DescribeClusterResult describeClusterResult = new DescribeClusterResult().withCluster(cluster);

    // Create a describe cluster request.
    DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(EMR_CLUSTER_ID);

    // Build the EMR cluster cache key
    EmrClusterCacheKey emrClusterCacheKey = new EmrClusterCacheKey(EMR_CLUSTER_NAME.toUpperCase(), accountId);

    // Build the EMR cluster cache
    Map<EmrClusterCacheKey, String> emrClusterCache = new ConcurrentHashMap<>();
    emrClusterCache.put(emrClusterCacheKey, EMR_CLUSTER_ID);

    // Mock the external calls.
    if (accountId == null)
    {
        when(emrClusterCacheMap.get(EMR_CLUSTER_CACHE_MAP_DEFAULT_AWS_ACCOUNT_ID_KEY)).thenReturn(emrClusterCache);
    }
    else
    {
        when(emrClusterCacheMap.get(accountId)).thenReturn(emrClusterCache);
    }

    when(emrOperations.describeClusterRequest(eq(amazonElasticMapReduceClient), any(DescribeClusterRequest.class))).thenReturn(describeClusterResult);
    when(configurationHelper.getProperty(ConfigurationValue.EMR_VALID_STATES)).thenReturn(ConfigurationValue.EMR_VALID_STATES.getDefaultValue().toString());
    when(configurationHelper.getProperty(ConfigurationValue.FIELD_DATA_DELIMITER))
        .thenReturn((String) ConfigurationValue.FIELD_DATA_DELIMITER.getDefaultValue());
    when(awsClientFactory.getEmrClient(awsParamsDto)).thenReturn(amazonElasticMapReduceClient);
    when(emrOperations.listEmrClusters(any(AmazonElasticMapReduceClient.class), any(ListClustersRequest.class))).thenReturn(listClusterResult);

    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, accountId, awsParamsDto);

    // Verify the external calls.
    verify(emrOperations).describeClusterRequest(eq(amazonElasticMapReduceClient), eq(describeClusterRequest));

    if (cluster == null)
    {
        verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
        verify(emrOperations).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    }
    else if (cluster.getStatus().getState().equals(EMR_INVALID_STATE))
    {
        verify(configurationHelper, times(2)).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper, times(2)).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
        verify(emrOperations).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    }
    else
    {
        verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory).getEmrClient(awsParamsDto);
    }

    verifyNoMoreInteractionsHelper();

    // Validate the results.
    assertEquals(clusterSummary, result);
}
 

/**
 * Get EMR cluster by cluster Id.
 *
 * @param clusterId the job Id returned by EMR for the cluster.
 * @param awsParams AWS related parameters for access/secret keys and proxy details.
 *
 * @return the cluster status.
 */
public Cluster getEmrClusterById(String clusterId, AwsParamsDto awsParams);