Java Code Examples for org.apache.hadoop.yarn.api.records.timeline.TimelineEntity#getOtherInfo()

The following examples show how to use org.apache.hadoop.yarn.api.records.timeline.TimelineEntity#getOtherInfo() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestHistoryEventTimelineConversion.java From tez with Apache License 2.0

5 votes

@Test(timeout = 5000)
public void testConvertAMLaunchedEvent() {
  long launchTime = random.nextLong();
  long submitTime = random.nextLong();
  AMLaunchedEvent event = new AMLaunchedEvent(applicationAttemptId, launchTime, submitTime, user);

  List<TimelineEntity> entities = HistoryEventTimelineConversion.convertToTimelineEntities(event);
  Assert.assertEquals(1, entities.size());
  TimelineEntity timelineEntity = entities.get(0);

  Assert.assertEquals("tez_" + applicationAttemptId.toString(), timelineEntity.getEntityId());
  Assert.assertEquals(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), timelineEntity.getEntityType());

  final Map<String, Set<String>> relatedEntities = timelineEntity.getRelatedEntities();
  Assert.assertEquals(0, relatedEntities.size());

  final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters();
  Assert.assertEquals(2, primaryFilters.size());
  Assert.assertTrue(primaryFilters.get(ATSConstants.USER).contains(user));
  Assert.assertTrue(primaryFilters.get(ATSConstants.APPLICATION_ID)
      .contains(applicationId.toString()));

  Assert.assertEquals(launchTime, timelineEntity.getStartTime().longValue());

  Assert.assertEquals(1, timelineEntity.getEvents().size());
  TimelineEvent evt = timelineEntity.getEvents().get(0);
  Assert.assertEquals(HistoryEventType.AM_LAUNCHED.name(), evt.getEventType());
  Assert.assertEquals(launchTime, evt.getTimestamp());

  final Map<String, Object> otherInfo = timelineEntity.getOtherInfo();
  Assert.assertEquals(4, otherInfo.size());
  Assert.assertEquals(submitTime, otherInfo.get(ATSConstants.APP_SUBMIT_TIME));
  Assert.assertEquals(applicationId.toString(), otherInfo.get(ATSConstants.APPLICATION_ID));
  Assert.assertEquals(applicationAttemptId.toString(), otherInfo.get(ATSConstants.APPLICATION_ATTEMPT_ID));
  Assert.assertEquals(user, otherInfo.get(ATSConstants.USER));
}

Example 2

Source File: TestHistoryEventTimelineConversion.java From tez with Apache License 2.0

5 votes

@Test(timeout = 5000)
public void testConvertContainerStoppedEvent() {
  long stopTime = random.nextLong();
  int exitStatus = random.nextInt();
  ContainerStoppedEvent event = new ContainerStoppedEvent(containerId, stopTime, exitStatus,
      applicationAttemptId);
  List<TimelineEntity> entities = HistoryEventTimelineConversion.convertToTimelineEntities(event);
  Assert.assertEquals(1, entities.size());
  TimelineEntity timelineEntity = entities.get(0);

  Assert.assertEquals("tez_" + containerId.toString(), timelineEntity.getEntityId());
  Assert.assertEquals(EntityTypes.TEZ_CONTAINER_ID.name(), timelineEntity.getEntityType());

  final Map<String, Set<String>> relatedEntities = timelineEntity.getRelatedEntities();
  Assert.assertEquals(1, relatedEntities.size());
  Assert.assertTrue(relatedEntities.get(EntityTypes.TEZ_APPLICATION_ATTEMPT.name())
      .contains("tez_" + applicationAttemptId.toString()));

  final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters();
  Assert.assertEquals(2, primaryFilters.size());
  Assert.assertTrue(primaryFilters.get(ATSConstants.APPLICATION_ID)
      .contains(applicationId.toString()));
  Assert.assertTrue(primaryFilters.get(ATSConstants.EXIT_STATUS).contains(exitStatus));

  Assert.assertEquals(1, timelineEntity.getEvents().size());
  final TimelineEvent evt = timelineEntity.getEvents().get(0);
  Assert.assertEquals(HistoryEventType.CONTAINER_STOPPED.name(), evt.getEventType());
  Assert.assertEquals(stopTime, evt.getTimestamp());

  final Map<String, Object> otherInfo = timelineEntity.getOtherInfo();
  Assert.assertEquals(2, otherInfo.size());
  Assert.assertEquals(exitStatus, otherInfo.get(ATSConstants.EXIT_STATUS));
  Assert.assertEquals(stopTime, otherInfo.get(ATSConstants.FINISH_TIME));
}

Example 3

Source File: TestHistoryEventTimelineConversion.java From tez with Apache License 2.0

5 votes

@Test(timeout = 5000)
public void testConvertDAGStartedEvent() {
  long startTime = random.nextLong();
  String dagName = "testDagName";
  DAGStartedEvent event = new DAGStartedEvent(tezDAGID, startTime, user, dagName);
  List<TimelineEntity> entities = HistoryEventTimelineConversion.convertToTimelineEntities(event);
  Assert.assertEquals(1, entities.size());
  TimelineEntity timelineEntity = entities.get(0);


  Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId());
  Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType());

  Assert.assertEquals(1, timelineEntity.getEvents().size());
  TimelineEvent evt = timelineEntity.getEvents().get(0);
  Assert.assertEquals(HistoryEventType.DAG_STARTED.name(), evt.getEventType());
  Assert.assertEquals(startTime, evt.getTimestamp());

  final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters();
  Assert.assertEquals(3, primaryFilters.size());
  Assert.assertTrue(primaryFilters.get(ATSConstants.USER).contains(user));
  Assert.assertTrue(primaryFilters.get(ATSConstants.APPLICATION_ID)
      .contains(applicationId.toString()));
  Assert.assertTrue(primaryFilters.get(ATSConstants.DAG_NAME).contains(dagName));

  final Map<String, Object> otherInfo = timelineEntity.getOtherInfo();
  Assert.assertEquals(2, otherInfo.size());
  Assert.assertEquals(startTime, otherInfo.get(ATSConstants.START_TIME));
  Assert.assertEquals(DAGState.RUNNING.name(), otherInfo.get(ATSConstants.STATUS));
}

Example 4

Source File: ApplicationHistoryManagerOnTimelineStore.java From big-c with Apache License 2.0

4 votes

private static ContainerReport convertToContainerReport(
    TimelineEntity entity, String serverHttpAddress, String user) {
  int allocatedMem = 0;
  int allocatedVcore = 0;
  String allocatedHost = null;
  int allocatedPort = -1;
  int allocatedPriority = 0;
  long createdTime = 0;
  long finishedTime = 0;
  String diagnosticsInfo = null;
  int exitStatus = ContainerExitStatus.INVALID;
  ContainerState state = null;
  String nodeHttpAddress = null;
  Map<String, Object> entityInfo = entity.getOtherInfo();
  if (entityInfo != null) {
    if (entityInfo
        .containsKey(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO)) {
      allocatedMem = (Integer) entityInfo.get(
              ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO);
    }
    if (entityInfo
        .containsKey(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO)) {
      allocatedVcore = (Integer) entityInfo.get(
              ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO);
    }
    if (entityInfo
        .containsKey(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO)) {
      allocatedHost =
          entityInfo
              .get(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO)
              .toString();
    }
    if (entityInfo
        .containsKey(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO)) {
      allocatedPort = (Integer) entityInfo.get(
              ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO);
    }
    if (entityInfo
        .containsKey(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO)) {
      allocatedPriority = (Integer) entityInfo.get(
              ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO);
    }
    if (entityInfo.containsKey(
        ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO)) {
      nodeHttpAddress =
          (String) entityInfo
            .get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO);
    }
  }
  List<TimelineEvent> events = entity.getEvents();
  if (events != null) {
    for (TimelineEvent event : events) {
      if (event.getEventType().equals(
          ContainerMetricsConstants.CREATED_EVENT_TYPE)) {
        createdTime = event.getTimestamp();
      } else if (event.getEventType().equals(
          ContainerMetricsConstants.FINISHED_EVENT_TYPE)) {
        finishedTime = event.getTimestamp();
        Map<String, Object> eventInfo = event.getEventInfo();
        if (eventInfo == null) {
          continue;
        }
        if (eventInfo
            .containsKey(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) {
          diagnosticsInfo =
              eventInfo.get(
                  ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)
                  .toString();
        }
        if (eventInfo
            .containsKey(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO)) {
          exitStatus = (Integer) eventInfo.get(
                  ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO);
        }
        if (eventInfo
            .containsKey(ContainerMetricsConstants.STATE_EVENT_INFO)) {
          state =
              ContainerState.valueOf(eventInfo.get(
                  ContainerMetricsConstants.STATE_EVENT_INFO).toString());
        }
      }
    }
  }
  NodeId allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort);
  ContainerId containerId =
      ConverterUtils.toContainerId(entity.getEntityId());
  String logUrl = WebAppUtils.getAggregatedLogURL(
      serverHttpAddress,
      allocatedNode.toString(),
      containerId.toString(),
      containerId.toString(),
      user);
  return ContainerReport.newInstance(
      ConverterUtils.toContainerId(entity.getEntityId()),
      Resource.newInstance(allocatedMem, allocatedVcore),
      NodeId.newInstance(allocatedHost, allocatedPort),
      Priority.newInstance(allocatedPriority),
      createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state,
      nodeHttpAddress);
}

Example 5

Source File: TestHistoryEventTimelineConversion.java From tez with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testConvertTaskAttemptFinishedEvent() {
  String vertexName = "testVertex";
  long creationTime = random.nextLong();
  long startTime = creationTime + 1000;
  long allocationTime = creationTime + 1001;
  long finishTime = startTime + 1002;
  TaskAttemptState state = TaskAttemptState
      .values()[random.nextInt(TaskAttemptState.values().length)];
  TaskAttemptTerminationCause error = TaskAttemptTerminationCause
      .values()[random.nextInt(TaskAttemptTerminationCause.values().length)];
  String diagnostics = "random diagnostics message";
  TezCounters counters = new TezCounters();
  long lastDataEventTime = finishTime - 1;
  List<DataEventDependencyInfo> events = Lists.newArrayList();
  events.add(new DataEventDependencyInfo(lastDataEventTime, tezTaskAttemptID));
  events.add(new DataEventDependencyInfo(lastDataEventTime, tezTaskAttemptID));

  TaskAttemptFinishedEvent event = new TaskAttemptFinishedEvent(tezTaskAttemptID, vertexName,
      startTime, finishTime, state, TaskFailureType.FATAL, error, diagnostics, counters, events, null, creationTime,
      tezTaskAttemptID, allocationTime, containerId, nodeId, "inProgressURL", "logsURL", "nodeHttpAddress");
  List<TimelineEntity> entities = HistoryEventTimelineConversion.convertToTimelineEntities(event);
  Assert.assertEquals(1, entities.size());
  TimelineEntity timelineEntity = entities.get(0);

  Assert.assertEquals(tezTaskAttemptID.toString(), timelineEntity.getEntityId());
  Assert.assertEquals(EntityTypes.TEZ_TASK_ATTEMPT_ID.name(), timelineEntity.getEntityType());

  final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters();
  Assert.assertEquals(5, primaryFilters.size());
  Assert.assertTrue(primaryFilters.get(ATSConstants.APPLICATION_ID)
      .contains(applicationId.toString()));
  Assert.assertTrue(primaryFilters.get(EntityTypes.TEZ_DAG_ID.name())
      .contains(tezDAGID.toString()));
  Assert.assertTrue(primaryFilters.get(EntityTypes.TEZ_VERTEX_ID.name())
      .contains(tezVertexID.toString()));
  Assert.assertTrue(primaryFilters.get(EntityTypes.TEZ_TASK_ID.name())
      .contains(tezTaskID.toString()));
  Assert.assertTrue(primaryFilters.get(ATSConstants.STATUS).contains(state.toString()));

  Assert.assertEquals(1, timelineEntity.getEvents().size());
  TimelineEvent evt = timelineEntity.getEvents().get(0);
  Assert.assertEquals(HistoryEventType.TASK_ATTEMPT_FINISHED.name(), evt.getEventType());
  Assert.assertEquals(finishTime, evt.getTimestamp());

  final Map<String, Object> otherInfo = timelineEntity.getOtherInfo();
  Assert.assertEquals(17, otherInfo.size());
  Assert.assertEquals(tezTaskAttemptID.toString(), 
      timelineEntity.getOtherInfo().get(ATSConstants.CREATION_CAUSAL_ATTEMPT));
  Assert.assertEquals(creationTime, timelineEntity.getOtherInfo().get(ATSConstants.CREATION_TIME));
  Assert.assertEquals(allocationTime, timelineEntity.getOtherInfo().get(ATSConstants.ALLOCATION_TIME));
  Assert.assertEquals(startTime, timelineEntity.getOtherInfo().get(ATSConstants.START_TIME));
  Assert.assertEquals(finishTime, otherInfo.get(ATSConstants.FINISH_TIME));
  Assert.assertEquals(finishTime - startTime, otherInfo.get(ATSConstants.TIME_TAKEN));
  Assert.assertEquals(state.name(), otherInfo.get(ATSConstants.STATUS));
  Assert.assertEquals(TaskFailureType.FATAL.name(), otherInfo.get(ATSConstants.TASK_FAILURE_TYPE));
  Assert.assertEquals(error.name(), otherInfo.get(ATSConstants.TASK_ATTEMPT_ERROR_ENUM));
  Assert.assertEquals(diagnostics, otherInfo.get(ATSConstants.DIAGNOSTICS));
  Map<String, Object> obj1 = (Map<String, Object>)otherInfo.get(ATSConstants.LAST_DATA_EVENTS);
  List<Object> obj2 = (List<Object>) obj1.get(ATSConstants.LAST_DATA_EVENTS);
  Assert.assertEquals(2, obj2.size());
  Map<String, Object> obj3 = (Map<String, Object>) obj2.get(0);
  Assert.assertEquals(events.get(0).getTimestamp(), obj3.get(ATSConstants.TIMESTAMP));
  Assert.assertTrue(otherInfo.containsKey(ATSConstants.COUNTERS));
  Assert.assertEquals("inProgressURL", otherInfo.get(ATSConstants.IN_PROGRESS_LOGS_URL));
  Assert.assertEquals("logsURL", otherInfo.get(ATSConstants.COMPLETED_LOGS_URL));
  Assert.assertEquals(nodeId.toString(), otherInfo.get(ATSConstants.NODE_ID));
  Assert.assertEquals(containerId.toString(), otherInfo.get(ATSConstants.CONTAINER_ID));
  Assert.assertEquals("nodeHttpAddress", otherInfo.get(ATSConstants.NODE_HTTP_ADDRESS));

  TaskAttemptFinishedEvent eventWithNullFailureType =
      new TaskAttemptFinishedEvent(tezTaskAttemptID, vertexName,
          startTime, finishTime, state, null, error, diagnostics, counters, events, null,
          creationTime,
          tezTaskAttemptID, allocationTime, containerId, nodeId, "inProgressURL", "logsURL",
          "nodeHttpAddress");
  List<TimelineEntity> evtEntities = HistoryEventTimelineConversion.convertToTimelineEntities(
      eventWithNullFailureType);
  Assert.assertEquals(1, evtEntities.size());
  TimelineEntity timelineEntityWithNullFailureType = evtEntities.get(0);

  Assert.assertNull(
      timelineEntityWithNullFailureType.getOtherInfo().get(ATSConstants.TASK_FAILURE_TYPE));
}

Example 6

Source File: TestHistoryEventTimelineConversion.java From tez with Apache License 2.0

4 votes

@Test(timeout = 5000)
public void testConvertTaskFinishedEvent() {
  String vertexName = "testVertexName";
  long startTime = random.nextLong();
  long finishTime = random.nextLong();
  TaskState state = TaskState.values()[random.nextInt(TaskState.values().length)];
  String diagnostics = "diagnostics message";
  TezCounters counters = new TezCounters();

  TaskFinishedEvent event = new TaskFinishedEvent(tezTaskID, vertexName, startTime, finishTime,
      tezTaskAttemptID, state, diagnostics, counters, 3);
  List<TimelineEntity> entities = HistoryEventTimelineConversion.convertToTimelineEntities(event);
  Assert.assertEquals(1, entities.size());
  TimelineEntity timelineEntity = entities.get(0);

  Assert.assertEquals(tezTaskID.toString(), timelineEntity.getEntityId());
  Assert.assertEquals(EntityTypes.TEZ_TASK_ID.name(), timelineEntity.getEntityType());

  final Map<String, Set<Object>> primaryFilters = timelineEntity.getPrimaryFilters();
  Assert.assertEquals(4, primaryFilters.size());
  Assert.assertTrue(primaryFilters.get(ATSConstants.APPLICATION_ID)
      .contains(applicationId.toString()));
  Assert.assertTrue(primaryFilters.get(EntityTypes.TEZ_DAG_ID.name())
      .contains(tezDAGID.toString()));
  Assert.assertTrue(primaryFilters.get(EntityTypes.TEZ_VERTEX_ID.name())
      .contains(tezVertexID.toString()));
  Assert.assertTrue(primaryFilters.get(ATSConstants.STATUS).contains(state.name()));

  Assert.assertEquals(1, timelineEntity.getEvents().size());
  TimelineEvent evt = timelineEntity.getEvents().get(0);
  Assert.assertEquals(HistoryEventType.TASK_FINISHED.name(), evt.getEventType());
  Assert.assertEquals(finishTime, evt.getTimestamp());

  final Map<String, Object> otherInfo = timelineEntity.getOtherInfo();
  Assert.assertEquals(7, otherInfo.size());
  Assert.assertEquals(finishTime, otherInfo.get(ATSConstants.FINISH_TIME));
  Assert.assertEquals(finishTime - startTime, otherInfo.get(ATSConstants.TIME_TAKEN));
  Assert.assertEquals(state.name(), otherInfo.get(ATSConstants.STATUS));
  Assert.assertEquals(tezTaskAttemptID.toString(),
      otherInfo.get(ATSConstants.SUCCESSFUL_ATTEMPT_ID));
  Assert.assertEquals(3, otherInfo.get(ATSConstants.NUM_FAILED_TASKS_ATTEMPTS));
  Assert.assertEquals(diagnostics, otherInfo.get(ATSConstants.DIAGNOSTICS));
  Assert.assertTrue(otherInfo.containsKey(ATSConstants.COUNTERS));
}