com.amazonaws.services.elasticmapreduce.model.ClusterStatus Java Exaples

Source File: TestEmrClusterJob.java From datacollector with Apache License 2.0

6 votes

@Test
public void testGetClusterStatus() {
  Properties properties = new Properties();
  EmrClusterJob emrClusterJob = new EmrClusterJob();
  EmrClusterJob.Client client = Mockito.spy(emrClusterJob.getClient(properties));
  AmazonElasticMapReduce emr = Mockito.mock(AmazonElasticMapReduce.class);
  Mockito.doReturn(emr).when(client).getEmrClient(Mockito.any(EmrClusterConfig.class));
  DescribeClusterResult result = Mockito.mock(DescribeClusterResult.class);
  Mockito.doReturn(result).when(emr).describeCluster(Mockito.any(DescribeClusterRequest
      .class));
  Cluster cluster = Mockito.mock(Cluster.class);
  Mockito.doReturn(cluster).when(result).getCluster();
  Mockito.doReturn(Mockito.mock(ClusterStatus.class)).when(cluster).getStatus();
  client.getClusterStatus("foo");
  Mockito.verify(emr, Mockito.times(1)).describeCluster(Mockito.any(DescribeClusterRequest
      .class));
  Mockito.verify(client, Mockito.times(1)).getEmrClient(Mockito.any(EmrClusterConfig.class));
}

Source File: EmrHelperTest.java From herd with Apache License 2.0

6 votes

@Test
public void testGetActiveEmrClusterIdAssertReturnActualClusterIdWhenClusterIdSpecifiedAndClusterStateActiveAndNameMatch()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId, emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null));

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId.trim()), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertParametersCaseIgnored()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId,
            emrHelper.getActiveEmrClusterId(StringUtils.upperCase(emrClusterId), StringUtils.upperCase(emrClusterName), null));

        verify(mockEmrDao).getEmrClusterById(eq(StringUtils.upperCase(emrClusterId)), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Source File: EmrClusterJob.java From datacollector with Apache License 2.0

5 votes

@Override
public Properties getClusterStatus(String clusterId) {
  DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest()
      .withClusterId(clusterId);
  DescribeClusterResult result= getEmrClient(emrClusterConfig).describeCluster(describeClusterRequest);
  EmrState emrState = new EmrState();
  ClusterStatus clusterStatus = result.getCluster().getStatus();
  emrState.setState(clusterStatus.getState());
  if (clusterStatus.getStateChangeReason() != null) {
    emrState.setMessage(clusterStatus.getStateChangeReason().getCode() + ":" + clusterStatus.getStateChangeReason
        ().getMessage());
  }
  LOG.debug("State of cluster: {} is {}", clusterId, clusterStatus.getState());
  return emrState.toProperties();
}

Source File: EmrDaoImplTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterByNameAndAccountIdWhenClusterNameIsInCacheWithInvalidStateWithNullAccountId()
{
    // Create a cluster with an invalid state.
    Cluster cluster = new Cluster().withStatus(new ClusterStatus().withState(EMR_INVALID_STATE));

    // Test that the EMR Cluster Cache is not used.
    getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(cluster, null);
}

Source File: EmrDaoImplTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterByNameAndAccountIdWhenClusterNameIsInCacheWithInvalidState()
{
    // Create a cluster with an invalid state.
    Cluster cluster = new Cluster().withStatus(new ClusterStatus().withState(EMR_INVALID_STATE));

    // Test that the EMR Cluster Cache is not used.
    getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(cluster, AWS_ACCOUNT_ID);
}

Source File: EmrDaoImplTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterByNameAndAccountIdWhenClusterNameIsInCacheWithNullAccountId()
{
    // Create a cluster with a valid state.
    Cluster cluster = new Cluster().withStatus(new ClusterStatus().withState("STARTING"));

    // Test the EMR Cluster Cache is used.
    getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(cluster, null);
}

Source File: EmrDaoImplTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterByNameAndAccountIdWhenClusterNameIsInCache()
{
    // Create a cluster with a valid state.
    Cluster cluster = new Cluster().withStatus(new ClusterStatus().withState("STARTING"));

    // Test the EMR Cluster Cache is used.
    getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(cluster, AWS_ACCOUNT_ID);
}

Source File: MockEmrOperationsImpl.java From herd with Apache License 2.0

5 votes

@Override
public ListClustersResult listEmrClusters(AmazonElasticMapReduceClient emrClient, ListClustersRequest listClustersRequest)
{
    List<ClusterSummary> clusterSummaryList = new ArrayList<>();
    for (MockEmrJobFlow cluster : emrClusters.values())
    {
        if (!listClustersRequest.getClusterStates().isEmpty() && listClustersRequest.getClusterStates().contains(cluster.getStatus()))
        {
            ClusterSummary clusterSummary = new ClusterSummary();
            clusterSummary.withId(cluster.getJobFlowId()).withName(cluster.getJobFlowName()).withStatus(new ClusterStatus().withState(cluster.getStatus())
                .withStateChangeReason(new ClusterStateChangeReason().withCode(cluster.getStatusChangeReason().getCode())
                    .withMessage(cluster.getStatusChangeReason().getMessage())).withTimeline(new ClusterTimeline().withCreationDateTime(
                    cluster.getStatusTimeline().getCreationTime() != null ? cluster.getStatusTimeline().getCreationTime().toGregorianCalendar().getTime() :
                        null).withEndDateTime(
                    cluster.getStatusTimeline().getEndTime() != null ? cluster.getStatusTimeline().getEndTime().toGregorianCalendar().getTime() : null)
                    .withReadyDateTime(
                        cluster.getStatusTimeline().getReadyTime() != null ? cluster.getStatusTimeline().getReadyTime().toGregorianCalendar().getTime() :
                            null)));
            clusterSummaryList.add(clusterSummary);
        }
    }
    if (StringUtils.isBlank(listClustersRequest.getMarker()))
    {
        return new ListClustersResult().withClusters(clusterSummaryList).withMarker(MOCK_EMR_MAKER);
    }
    else
    {
        return new ListClustersResult().withClusters(clusterSummaryList);
    }
}

Source File: EmrClusterTableProviderTest.java From aws-athena-query-federation with Apache License 2.0

5 votes

private ClusterSummary makeClusterSummary(String id)
{
    return new ClusterSummary()
            .withName("name")
            .withId(id)
            .withStatus(new ClusterStatus()
                    .withState("state")
                    .withStateChangeReason(new ClusterStateChangeReason()
                            .withCode("state_code")
                            .withMessage("state_msg")))
            .withNormalizedInstanceHours(100);
}

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertParametersTrimmed()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId,
            emrHelper.getActiveEmrClusterId(StringUtils.wrap(emrClusterId, BLANK_TEXT), StringUtils.wrap(emrClusterName, BLANK_TEXT), null));

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId.trim()), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertErrorWhenClusterIdSpecifiedAndClusterStateNotActive()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        ClusterState actualClusterState = ClusterState.TERMINATED;
        when(mockEmrDao.getEmrClusterById(any(), any()))
            .thenReturn(new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(actualClusterState)));

        try
        {
            emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null);
            fail();
        }
        catch (IllegalArgumentException e)
        {
            assertEquals(String.format("The cluster with ID \"%s\" is not active. The cluster state must be in one of [STARTING, BOOTSTRAPPING, RUNNING, " +
                "WAITING]. Current state is \"%s\"", emrClusterId, actualClusterState), e.getMessage());
        }

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertReturnActualClusterIdWhenClusterStateActiveAndNameNotSpecified()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = null;
        String expectedEmrClusterId = "expectedEmrClusterId";
        String actualEmrClusterName = "actualEmrClusterName";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(actualEmrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId, emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null));

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertErrorWhenClusterIdSpecifiedAndNameMismatch()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";
        String actualEmrClusterName = "actualEmrClusterName";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(actualEmrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        try
        {
            emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null);
            fail();
        }
        catch (IllegalArgumentException e)
        {
            assertEquals(String
                .format("The cluster with ID \"%s\" does not match the expected name \"%s\". The actual name is \"%s\".", expectedEmrClusterId,
                    emrClusterName, actualEmrClusterName), e.getMessage());
        }

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId.trim()), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Source File: EmrServiceImpl.java From herd with Apache License 2.0

5 votes

/**
 * Updates EMR cluster model object with the specified EMR cluster status information.
 *
 * @param emrCluster the EMR cluster
 * @param clusterStatus the EMR cluster status information
 */
private void setEmrClusterStatus(EmrCluster emrCluster, ClusterStatus clusterStatus)
{
    // Log cluster status information.
    LOGGER.info("emrClusterId=\"{}\" emrClusterStatus={}", emrCluster.getId(), jsonHelper.objectToJson(clusterStatus));

    // Update the EMR cluster with the status information.
    emrCluster.setStatus(clusterStatus.getState());
    emrCluster
        .setStatusChangeReason(new StatusChangeReason(clusterStatus.getStateChangeReason().getCode(), clusterStatus.getStateChangeReason().getMessage()));
    emrCluster.setStatusTimeline(new StatusTimeline(toXmlGregorianCalendar(clusterStatus.getTimeline().getCreationDateTime()),
        toXmlGregorianCalendar(clusterStatus.getTimeline().getReadyDateTime()), toXmlGregorianCalendar(clusterStatus.getTimeline().getEndDateTime())));
}

Source File: EmrOperatorFactory.java From digdag with Apache License 2.0

4 votes

private Optional<String> checkClusterBootStatus(AmazonElasticMapReduce emr, NewCluster cluster, TaskState state)
{
    // Only creating a cluster, with no steps?
    boolean createOnly = cluster.steps() == 0;

    DescribeClusterResult describeClusterResult = pollingRetryExecutor(state, "describe-cluster")
            .withRetryInterval(DurationInterval.of(Duration.ofSeconds(30), Duration.ofMinutes(5)))
            .retryUnless(AmazonServiceException.class, Aws::isDeterministicException)
            .run(ds -> emr.describeCluster(new DescribeClusterRequest().withClusterId(cluster.id())));

    ClusterStatus clusterStatus = describeClusterResult.getCluster().getStatus();
    String clusterState = clusterStatus.getState();

    switch (clusterState) {
        case "STARTING":
            logger.info("EMR cluster starting: {}", cluster.id());
            return Optional.absent();
        case "BOOTSTRAPPING":
            logger.info("EMR cluster bootstrapping: {}", cluster.id());
            return Optional.absent();

        case "RUNNING":
        case "WAITING":
            logger.info("EMR cluster up: {}", cluster.id());
            return Optional.of(clusterState);

        case "TERMINATED_WITH_ERRORS":
            if (createOnly) {
                // TODO: log more information about the errors
                // TODO: inspect state change reason to figure out whether it was the boot that failed or e.g. steps submitted by another agent
                throw new TaskExecutionException("EMR boot failed: " + cluster.id());
            }
            return Optional.of(clusterState);

        case "TERMINATING":
            if (createOnly) {
                // Keep waiting for the final state
                // TODO: inspect state change reason and bail early here
                return Optional.absent();
            }
            return Optional.of(clusterState);

        case "TERMINATED":
            return Optional.of(clusterState);

        default:
            throw new RuntimeException("Unknown EMR cluster state: " + clusterState);
    }
}

Source File: EmrHelperServiceImplTest.java From herd with Apache License 2.0

4 votes

@Test
public void testEmrCreateClusterAwsSpecificStepsImpl()
{
    // Create an AWS params DTO
    AwsParamsDto awsParamsDto = new AwsParamsDto();

    // Create an EMR cluster definition object
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    emrClusterDefinition.setAccountId(AWS_ACCOUNT_ID);
    emrClusterDefinition.setInstanceDefinitions(new InstanceDefinitions());

    // Create an EMR cluster create request
    EmrClusterCreateRequest emrClusterCreateRequest =
        new EmrClusterCreateRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, NO_DRY_RUN, emrClusterDefinition);
    emrClusterCreateRequest.setEmrClusterDefinitionOverride(emrClusterDefinition);

    // Create an EMR cluster alternate key DTO
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = new EmrClusterAlternateKeyDto(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME);

    // Create a cluster summary object
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId(EMR_CLUSTER_ID);
    clusterSummary.setStatus(new ClusterStatus().withState(EMR_CLUSTER_STATUS));

    // Mock the external calls.
    when(emrHelper.getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId())).thenReturn(awsParamsDto);
    when(emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions())).thenReturn(false);
    when(emrHelper.buildEmrClusterName(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(),
        emrClusterAlternateKeyDto.getEmrClusterName())).thenReturn(EMR_CLUSTER_NAME);
    when(emrDao.getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, emrClusterDefinition.getAccountId(), awsParamsDto)).thenReturn(clusterSummary);

    // Call the method under test.
    emrHelperServiceImpl.emrCreateClusterAwsSpecificSteps(emrClusterCreateRequest, emrClusterDefinition, emrClusterAlternateKeyDto);

    // Verify the external calls.
    verify(emrHelper).getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId());
    verify(emrHelper).isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions());
    verify(emrPricingHelper).updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto);
    verify(emrHelper).buildEmrClusterName(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(),
        emrClusterAlternateKeyDto.getEmrClusterName());
    verify(emrDao).getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, emrClusterDefinition.getAccountId(), awsParamsDto);
    verifyNoMoreInteractionsHelper();
}

com.amazonaws.services.elasticmapreduce.model.ClusterStatus Java Examples