com.amazonaws.services.elasticmapreduce.model.DescribeClusterRequest Java Examples

/**
 * Calls ListClusters and DescribeCluster on the AWS EMR Client returning all clusters that match the supplied
 * predicate and attempting to push down certain predicates (namely queries for specific cluster) to EC2.
 *
 * @See TableProvider
 */
@Override
public void readWithConstraint(BlockSpiller spiller, ReadRecordsRequest recordsRequest, QueryStatusChecker queryStatusChecker)
{
    boolean done = false;
    ListClustersRequest request = new ListClustersRequest();

    while (!done) {
        ListClustersResult response = emr.listClusters(request);

        for (ClusterSummary next : response.getClusters()) {
            Cluster cluster = null;
            if (!next.getStatus().getState().toLowerCase().contains("terminated")) {
                DescribeClusterResult clusterResponse = emr.describeCluster(new DescribeClusterRequest().withClusterId(next.getId()));
                cluster = clusterResponse.getCluster();
            }
            clusterToRow(next, cluster, spiller);
        }

        request.setMarker(response.getMarker());

        if (response.getMarker() == null || !queryStatusChecker.isQueryRunning()) {
            done = true;
        }
    }
}
 

@Override
public void afterPropertiesSet() throws Exception {
    createEmrClient();

    String emrInfo = new String(Files.readAllBytes(Paths.get(jobFlowJson)));
    JsonReader reader = Json.createReader(new StringReader(emrInfo));
    JsonObject object = reader.readObject();
    if(object.containsKey("jobFlowId"))
    {
        setClusterID(object.getString("jobFlowId"));
        logger.info("Cluster ID = "+ clusterID);
        DescribeClusterRequest req = new DescribeClusterRequest().withClusterId(clusterID);
        DescribeClusterResult result = emrClient.describeCluster(req);
        String name = result.getCluster().getName();
        logger.info("Cluster Name = "+ name);
        setClusterName(name.substring(name.lastIndexOf(".")+1));
    }

}
 

@Override
public Cluster getEmrClusterById(String clusterId, AwsParamsDto awsParams)
{
    Cluster cluster = null;
    if (StringUtils.isNotBlank(clusterId))
    {
        DescribeClusterResult describeClusterResult =
            emrOperations.describeClusterRequest(getEmrClient(awsParams), new DescribeClusterRequest().withClusterId(clusterId));
        if (describeClusterResult != null && describeClusterResult.getCluster() != null)
        {
            cluster = describeClusterResult.getCluster();
        }
    }

    return cluster;
}
 

@Test
public void testGetClusterStatus() {
  Properties properties = new Properties();
  EmrClusterJob emrClusterJob = new EmrClusterJob();
  EmrClusterJob.Client client = Mockito.spy(emrClusterJob.getClient(properties));
  AmazonElasticMapReduce emr = Mockito.mock(AmazonElasticMapReduce.class);
  Mockito.doReturn(emr).when(client).getEmrClient(Mockito.any(EmrClusterConfig.class));
  DescribeClusterResult result = Mockito.mock(DescribeClusterResult.class);
  Mockito.doReturn(result).when(emr).describeCluster(Mockito.any(DescribeClusterRequest
      .class));
  Cluster cluster = Mockito.mock(Cluster.class);
  Mockito.doReturn(cluster).when(result).getCluster();
  Mockito.doReturn(Mockito.mock(ClusterStatus.class)).when(cluster).getStatus();
  client.getClusterStatus("foo");
  Mockito.verify(emr, Mockito.times(1)).describeCluster(Mockito.any(DescribeClusterRequest
      .class));
  Mockito.verify(client, Mockito.times(1)).getEmrClient(Mockito.any(EmrClusterConfig.class));
}
 

protected List<String> getActiveTaggedClusters() throws Exception{
	AmazonElasticMapReduceClient emrClient = new AmazonElasticMapReduceClient();
	List<String> waitingClusters = new ArrayList<String>();
	ListClustersResult clusterResult = emrClient.listClusters(new ListClustersRequest().withClusterStates(ClusterState.WAITING));
	
	DescribeClusterRequest specifcTagDescribe = new DescribeClusterRequest();
	specifcTagDescribe.putCustomQueryParameter("Cluster.Tags",null);
	 for( ClusterSummary cluster : clusterResult.getClusters()){
		 	System.out.println("list cluster id "+cluster.getId());
		 	List<Tag> tagList = emrClient.describeCluster(specifcTagDescribe.withClusterId(cluster.getId())).getCluster().getTags();
		 	for(Tag tag:tagList){
		 		if(tag.getKey().equals(props.getProperty("edba.cluster.tag.key"))){
		 			waitingClusters.add(cluster.getId());
		 		}
		 	}
		 	
	}
	return waitingClusters;
	
}
 

/**
 * Fetch EMR info.
 *
 * @param temporaryCredentials the temporary credentials
 * @param skipRegions the skip regions
 * @param accountId the accountId
 * @param accountName the account name
 * @return the map
 */
public static Map<String,List<Cluster>> fetchEMRInfo(BasicSessionCredentials temporaryCredentials, String skipRegions,String accountId,String accountName){

	Map<String,List<Cluster>> clusterList = new LinkedHashMap<>();
	String expPrefix = InventoryConstants.ERROR_PREFIX_CODE+accountId + "\",\"Message\": \"Exception in fetching info for resource in specific region\" ,\"type\": \"EMR\" , \"region\":\"" ;
	for(Region region : RegionUtils.getRegions()){
		try{
			if(!skipRegions.contains(region.getName())){
				AmazonElasticMapReduce emrClient = AmazonElasticMapReduceClientBuilder.standard().
				 	withCredentials(new AWSStaticCredentialsProvider(temporaryCredentials)).withRegion(region.getName()).build();
				List<ClusterSummary> clusters = new ArrayList<>();
				String marker = null;
				ListClustersResult clusterResult ;
				do{
					clusterResult = emrClient.listClusters(new ListClustersRequest().withMarker(marker));
					clusters.addAll(clusterResult.getClusters());
					marker = clusterResult.getMarker();
				}while(marker!=null);

				List<Cluster> clustersList = new ArrayList<>();
				clusters.forEach(cluster ->
					{
						DescribeClusterResult descClstrRslt = emrClient.describeCluster(new DescribeClusterRequest().withClusterId(cluster.getId()));
						clustersList.add(descClstrRslt.getCluster());
					});

				if( !clustersList.isEmpty() ){
					log.debug(InventoryConstants.ACCOUNT + accountId +" Type : EMR "+region.getName() + " >> "+clustersList.size());
					clusterList.put(accountId+delimiter+accountName+delimiter+region.getName(),clustersList);
				}
			}
		}catch(Exception e){
			if(region.isServiceSupported(AmazonElasticMapReduce.ENDPOINT_PREFIX)){
				log.warn(expPrefix+ region.getName()+InventoryConstants.ERROR_CAUSE +e.getMessage()+"\"}");
				ErrorManageUtil.uploadError(accountId,region.getName(),"emr",e.getMessage());
			}
		}
	}
	return clusterList;
}
 

@VisibleForTesting
YarnClient getYarnClient(String clusterId, EmrClusterConfig emrClusterConfig) {
  DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(clusterId);
  DescribeClusterResult result= getEmrClient(emrClusterConfig).describeCluster(describeClusterRequest);
  String rmHostname =  result.getCluster().getMasterPublicDnsName();
  if (yarnClient == null) {
    yarnClient = YarnClient.createYarnClient();
    org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
    configuration.set("yarn.resourcemanager.hostname", rmHostname);
    yarnClient.init(configuration);
    yarnClient.start();
  }
  return yarnClient;
}
 

@Override
public Properties getClusterStatus(String clusterId) {
  DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest()
      .withClusterId(clusterId);
  DescribeClusterResult result= getEmrClient(emrClusterConfig).describeCluster(describeClusterRequest);
  EmrState emrState = new EmrState();
  ClusterStatus clusterStatus = result.getCluster().getStatus();
  emrState.setState(clusterStatus.getState());
  if (clusterStatus.getStateChangeReason() != null) {
    emrState.setMessage(clusterStatus.getStateChangeReason().getCode() + ":" + clusterStatus.getStateChangeReason
        ().getMessage());
  }
  LOG.debug("State of cluster: {} is {}", clusterId, clusterStatus.getState());
  return emrState.toProperties();
}
 

/**
 * Helper method to determine if an Amazon EMR cluster exists
 * 
 * @param client
 *        The {@link AmazonElasticMapReduceClient} with read permissions
 * @param clusterIdentifier
 *        The Amazon EMR cluster to check
 * @return true if the Amazon EMR cluster exists, otherwise false
 */
public static boolean clusterExists(AmazonElasticMapReduce client, String clusterIdentifier) {
	if (clusterIdentifier != null && !clusterIdentifier.isEmpty()) {
		ListClustersResult clustersList = client.listClusters();
		ListIterator<ClusterSummary> iterator = clustersList.getClusters().listIterator();
		ClusterSummary summary;
		for (summary = iterator.next() ; iterator.hasNext();summary = iterator.next()) {
			if (summary.getId().equals(clusterIdentifier)) {
				DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(clusterIdentifier);	
				DescribeClusterResult result = client.describeCluster(describeClusterRequest);	
				if (result != null) {
					Cluster cluster = result.getCluster();
					//check if HBase is installed on this cluster
					if (isHBaseInstalled(client, cluster.getId())) return false;
					String state = cluster.getStatus().getState();
					LOG.info(clusterIdentifier + " is " + state + ". ");
					if (state.equalsIgnoreCase("RUNNING") ||state.equalsIgnoreCase("WAITING"))	{
						LOG.info("The cluster with id " + clusterIdentifier + " exists and is " + state);   
						return true;
					}
				}
			}		
		}					
	}
	LOG.info("The cluster with id " + clusterIdentifier + " does not exist");
	return false;  
}
 

public boolean registerCluster() {
    String sql = "INSERT IGNORE INTO `EMR_CLUSTER` (`CLUSTER_NAME`,`CLUSTER_ID`, CREATE_TIME, STATUS) VALUES (?, ?, now(), 'R')";

    template.update(sql, clusterName, clusterID);

    String runningClusterID = template.queryForObject(SELECT_CLUSTER_ID_SQL, new Object[]{clusterName}, String.class);
    boolean continueProcess = false;

    if (!runningClusterID.equals(clusterID)) {
        //Verify the running cluster is still up

        DescribeClusterRequest req = new DescribeClusterRequest().withClusterId(runningClusterID);
        DescribeClusterResult result = emrClient.describeCluster(req);

        String clusterState = result.getCluster().getStatus().getState();

        if (clusterState.equals(ClusterState.TERMINATED.toString()) || clusterState.equals(ClusterState.TERMINATED_WITH_ERRORS.toString())) {
            continueProcess = prepareProcessing();
        } else if (clusterState.equals(ClusterState.TERMINATING.toString())) {
            //Wait till cluster is terminated
            logger.info(String.format("Running Cluster %s is in TERMINCATING state, waiting for it to be terminated", runningClusterID));

            do {
                try {
                    Thread.sleep(30000);
                } catch (InterruptedException ex) {

                }
                result = emrClient.describeCluster(req);
                clusterState = result.getCluster().getStatus().getState();
            } while (clusterState.equals(ClusterState.TERMINATING.toString()));

            if (clusterState.equals(ClusterState.TERMINATED.toString()) || clusterState.equals(ClusterState.TERMINATED_WITH_ERRORS)) {
                continueProcess = prepareProcessing();
            }

        } else {
            logger.info("Running cluster is " + runningClusterID + ", exiting");
        }

    } else {
        continueProcess = true;
    }

    return continueProcess;
}
 

public void clusterAutoScale()
  {
      logger.info("Auto Scale Job started");
      Map<String, Object> lastCheck = template.queryForMap(AUTO_SCALE_QUERY);
      long minutes = (Long) lastCheck.get("age");
      if(minutes >= autoScaleIntervalInMin) {
          long id = (Long)lastCheck.get("ID") + 1;
          int updated = template.update("INSERT IGNORE INTO METASTOR_EMR_AUTOSCALE (`ID`," +
                  "`CLUSTER_ID`) VALUES (?, ?)", id, clusterID);

          if (updated > 0) {

              int clusterNumber = calculateNumberOfClustersNeeded();
              template.update("UPDATE METASTOR_EMR_AUTOSCALE SET TOTAL_CLUSTER= ? WHERE ID=?", clusterNumber, id);
              if (clusterNumber <= 1) return;

              List<Map<String, Object>> clusters = template.queryForList("select * from EMR_CLUSTER");
              List<String> existingCluster = new ArrayList<>();

              createEmrClient();
              for (Map<String, Object> record : clusters) {
                  String cid = (String) record.get("CLUSTER_ID");
                  String cname = (String) record.get("CLUSTER_NAME");

                  boolean clusterIsAlive = true;

                  try {

                      DescribeClusterRequest req = new DescribeClusterRequest().withClusterId(cid);
                      DescribeClusterResult result = emrClient.describeCluster(req);

                      String clusterState = result.getCluster().getStatus().getState();
                      if (clusterState.equals(ClusterState.TERMINATED.toString()) || clusterState.equals(ClusterState.TERMINATED_WITH_ERRORS.toString())) {
                          deleteCluster(cid);
                          clusterIsAlive = false;
                      }
                  } catch (Exception ex) {
                      logger.warning("Error getting info for cluster " + cid + ex.getMessage());
                      clusterIsAlive = false;
                  }


                  if (clusterIsAlive) {
                      existingCluster.add(cname);
                  }

              }

              // Start Additional if required
		startAdditionalClusters( clusterNumber, existingCluster );
	}
      }
logger.info("Auto Scale Job completed");

  }
 

private void getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(Cluster cluster, String accountId)
{
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto =
        new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT,
            AWS_REGION_NAME_US_EAST_1);

    // Create a mock AmazonElasticMapReduceClient.
    AmazonElasticMapReduceClient amazonElasticMapReduceClient = mock(AmazonElasticMapReduceClient.class);

    // Create a cluster summary.
    ClusterSummary clusterSummary =
        new ClusterSummary().withName(EMR_CLUSTER_NAME).withId(EMR_CLUSTER_ID).withStatus(cluster == null ? null : cluster.getStatus());

    // Create a list cluster result with the matching cluster.
    ListClustersResult listClusterResult = new ListClustersResult().withClusters(clusterSummary);

    // Create a describe cluster result.
    DescribeClusterResult describeClusterResult = new DescribeClusterResult().withCluster(cluster);

    // Create a describe cluster request.
    DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(EMR_CLUSTER_ID);

    // Build the EMR cluster cache key
    EmrClusterCacheKey emrClusterCacheKey = new EmrClusterCacheKey(EMR_CLUSTER_NAME.toUpperCase(), accountId);

    // Build the EMR cluster cache
    Map<EmrClusterCacheKey, String> emrClusterCache = new ConcurrentHashMap<>();
    emrClusterCache.put(emrClusterCacheKey, EMR_CLUSTER_ID);

    // Mock the external calls.
    if (accountId == null)
    {
        when(emrClusterCacheMap.get(EMR_CLUSTER_CACHE_MAP_DEFAULT_AWS_ACCOUNT_ID_KEY)).thenReturn(emrClusterCache);
    }
    else
    {
        when(emrClusterCacheMap.get(accountId)).thenReturn(emrClusterCache);
    }

    when(emrOperations.describeClusterRequest(eq(amazonElasticMapReduceClient), any(DescribeClusterRequest.class))).thenReturn(describeClusterResult);
    when(configurationHelper.getProperty(ConfigurationValue.EMR_VALID_STATES)).thenReturn(ConfigurationValue.EMR_VALID_STATES.getDefaultValue().toString());
    when(configurationHelper.getProperty(ConfigurationValue.FIELD_DATA_DELIMITER))
        .thenReturn((String) ConfigurationValue.FIELD_DATA_DELIMITER.getDefaultValue());
    when(awsClientFactory.getEmrClient(awsParamsDto)).thenReturn(amazonElasticMapReduceClient);
    when(emrOperations.listEmrClusters(any(AmazonElasticMapReduceClient.class), any(ListClustersRequest.class))).thenReturn(listClusterResult);

    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, accountId, awsParamsDto);

    // Verify the external calls.
    verify(emrOperations).describeClusterRequest(eq(amazonElasticMapReduceClient), eq(describeClusterRequest));

    if (cluster == null)
    {
        verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
        verify(emrOperations).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    }
    else if (cluster.getStatus().getState().equals(EMR_INVALID_STATE))
    {
        verify(configurationHelper, times(2)).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper, times(2)).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
        verify(emrOperations).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    }
    else
    {
        verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory).getEmrClient(awsParamsDto);
    }

    verifyNoMoreInteractionsHelper();

    // Validate the results.
    assertEquals(clusterSummary, result);
}
 

private Optional<String> checkClusterBootStatus(AmazonElasticMapReduce emr, NewCluster cluster, TaskState state)
{
    // Only creating a cluster, with no steps?
    boolean createOnly = cluster.steps() == 0;

    DescribeClusterResult describeClusterResult = pollingRetryExecutor(state, "describe-cluster")
            .withRetryInterval(DurationInterval.of(Duration.ofSeconds(30), Duration.ofMinutes(5)))
            .retryUnless(AmazonServiceException.class, Aws::isDeterministicException)
            .run(ds -> emr.describeCluster(new DescribeClusterRequest().withClusterId(cluster.id())));

    ClusterStatus clusterStatus = describeClusterResult.getCluster().getStatus();
    String clusterState = clusterStatus.getState();

    switch (clusterState) {
        case "STARTING":
            logger.info("EMR cluster starting: {}", cluster.id());
            return Optional.absent();
        case "BOOTSTRAPPING":
            logger.info("EMR cluster bootstrapping: {}", cluster.id());
            return Optional.absent();

        case "RUNNING":
        case "WAITING":
            logger.info("EMR cluster up: {}", cluster.id());
            return Optional.of(clusterState);

        case "TERMINATED_WITH_ERRORS":
            if (createOnly) {
                // TODO: log more information about the errors
                // TODO: inspect state change reason to figure out whether it was the boot that failed or e.g. steps submitted by another agent
                throw new TaskExecutionException("EMR boot failed: " + cluster.id());
            }
            return Optional.of(clusterState);

        case "TERMINATING":
            if (createOnly) {
                // Keep waiting for the final state
                // TODO: inspect state change reason and bail early here
                return Optional.absent();
            }
            return Optional.of(clusterState);

        case "TERMINATED":
            return Optional.of(clusterState);

        default:
            throw new RuntimeException("Unknown EMR cluster state: " + clusterState);
    }
}
 

/**
 * Helper method to determine the Amazon EMR cluster state
 * 
 * @param client
 *        The {@link AmazonElasticMapReduceClient} with read permissions
 * @param clusterIdentifier
 *        The Amazon EMR cluster to get the state of - e.g. j-2A98VJHDSU48M
 * @return The String representation of the Amazon EMR cluster state
 */
public static String clusterState(AmazonElasticMapReduce client, String clusterIdentifier) {
	DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(clusterIdentifier);
	DescribeClusterResult result= client.describeCluster(describeClusterRequest);
	if (result != null) {
		return result.getCluster().getStatus().getState();
	}
	return null;
}