Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getRMAppAttempt()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getRMAppAttempt() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ResourceManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public void handle(RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptID = event.getApplicationAttemptId();
  ApplicationId appAttemptId = appAttemptID.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appAttemptId);
  if (rmApp != null) {
    RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptID);
    if (rmAppAttempt != null) {
      try {
        rmAppAttempt.handle(event);
      } catch (Throwable t) {
        LOG.error("Error in handling event type " + event.getType()
            + " for applicationAttempt " + appAttemptId, t);
      }
    }
  }
}
Example 2
Source File: MockRM.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public void waitForState(ApplicationAttemptId attemptId, 
                         RMAppAttemptState finalState)
    throws Exception {
  RMApp app = getRMContext().getRMApps().get(attemptId.getApplicationId());
  Assert.assertNotNull("app shouldn't be null", app);
  RMAppAttempt attempt = app.getRMAppAttempt(attemptId);
  int timeoutSecs = 0;
  while (!finalState.equals(attempt.getAppAttemptState()) && timeoutSecs++ < 40) {
    System.out.println("AppAttempt : " + attemptId 
        + " State is : " + attempt.getAppAttemptState()
        + " Waiting for state : " + finalState);
    Thread.sleep(1000);
  }
  System.out.println("Attempt State is : " + attempt.getAppAttemptState());
  Assert.assertEquals("Attempt state is not correct (timedout)", finalState,
      attempt.getAppAttemptState());
}
Example 3
Source File: MockAM.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public void waitForState(RMAppAttemptState finalState) throws Exception {
  RMApp app = context.getRMApps().get(attemptId.getApplicationId());
  RMAppAttempt attempt = app.getRMAppAttempt(attemptId);
  int timeoutSecs = 0;
  while (!finalState.equals(attempt.getAppAttemptState())
      && timeoutSecs++ < 40) {
    System.out
        .println("AppAttempt : " + attemptId + " State is : " 
            + attempt.getAppAttemptState()
            + " Waiting for state : " + finalState);
    Thread.sleep(1000);
  }
  System.out.println("AppAttempt State is : " + attempt.getAppAttemptState());
  Assert.assertEquals("AppAttempt state is not correct (timedout)",
      finalState, attempt.getAppAttemptState());
}
Example 4
Source File: ResourceManager.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public void handle(RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptID = event.getApplicationAttemptId();
  ApplicationId appAttemptId = appAttemptID.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appAttemptId);
  if (rmApp != null) {
    RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptID);
    if (rmAppAttempt != null) {
      try {
        rmAppAttempt.handle(event);
      } catch (Throwable t) {
        LOG.error("Error in handling event type " + event.getType()
            + " for applicationAttempt " + appAttemptId, t);
      }
    }
  }
}
Example 5
Source File: MockRM.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public void waitForState(ApplicationAttemptId attemptId, 
                         RMAppAttemptState finalState)
    throws Exception {
  RMApp app = getRMContext().getRMApps().get(attemptId.getApplicationId());
  Assert.assertNotNull("app shouldn't be null", app);
  RMAppAttempt attempt = app.getRMAppAttempt(attemptId);
  int timeoutSecs = 0;
  while (!finalState.equals(attempt.getAppAttemptState()) && timeoutSecs++ < 40) {
    System.out.println("AppAttempt : " + attemptId 
        + " State is : " + attempt.getAppAttemptState()
        + " Waiting for state : " + finalState);
    Thread.sleep(1000);
  }
  System.out.println("Attempt State is : " + attempt.getAppAttemptState());
  Assert.assertEquals("Attempt state is not correct (timedout)", finalState,
      attempt.getAppAttemptState());
}
Example 6
Source File: MockAM.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public void waitForState(RMAppAttemptState finalState) throws Exception {
  RMApp app = context.getRMApps().get(attemptId.getApplicationId());
  RMAppAttempt attempt = app.getRMAppAttempt(attemptId);
  int timeoutSecs = 0;
  while (!finalState.equals(attempt.getAppAttemptState())
      && timeoutSecs++ < 40) {
    System.out
        .println("AppAttempt : " + attemptId + " State is : " 
            + attempt.getAppAttemptState()
            + " Waiting for state : " + finalState);
    Thread.sleep(1000);
  }
  System.out.println("AppAttempt State is : " + attempt.getAppAttemptState());
  Assert.assertEquals("AppAttempt state is not correct (timedout)",
      finalState, attempt.getAppAttemptState());
}
Example 7
Source File: ResourceTrackerService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
Example 8
Source File: ResourceTrackerService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
Example 9
Source File: RMAppBlock.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
protected void createApplicationAttemptTable(Block html,
    Collection<ApplicationAttemptReport> attempts) {
  TBODY<TABLE<Hamlet>> tbody =
      html.table("#attempts").thead().tr().th(".id", "Attempt ID")
        .th(".started", "Started").th(".node", "Node").th(".logs", "Logs")
        ._()._().tbody();
  RMApp rmApp = this.rm.getRMContext().getRMApps().get(this.appID);
  if (rmApp == null) {
    return;
  }
  StringBuilder attemptsTableData = new StringBuilder("[\n");
  for (final ApplicationAttemptReport appAttemptReport : attempts) {
    RMAppAttempt rmAppAttempt =
        rmApp.getRMAppAttempt(appAttemptReport.getApplicationAttemptId());
    if (rmAppAttempt == null) {
      continue;
    }
    AppAttemptInfo attemptInfo =
        new AppAttemptInfo(rmAppAttempt, rmApp.getUser());
    String nodeLink = attemptInfo.getNodeHttpAddress();
    if (nodeLink != null) {
      nodeLink = WebAppUtils.getHttpSchemePrefix(conf) + nodeLink;
    }
    String logsLink = attemptInfo.getLogsLink();
    attemptsTableData
      .append("[\"<a href='")
      .append(url("appattempt", rmAppAttempt.getAppAttemptId().toString()))
      .append("'>")
      .append(String.valueOf(rmAppAttempt.getAppAttemptId()))
      .append("</a>\",\"")
      .append(attemptInfo.getStartTime())
      .append("\",\"<a ")
      .append(nodeLink == null ? "#" : "href='" + nodeLink)
      .append("'>")
      .append(
        nodeLink == null ? "N/A" : StringEscapeUtils
          .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))
      .append("</a>\",\"<a ")
      .append(logsLink == null ? "#" : "href='" + logsLink).append("'>")
      .append(logsLink == null ? "N/A" : "Logs").append("</a>\"],\n");
  }
  if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
    attemptsTableData.delete(attemptsTableData.length() - 2,
      attemptsTableData.length() - 1);
  }
  attemptsTableData.append("]");
  html.script().$type("text/javascript")
    ._("var attemptsTableData=" + attemptsTableData)._();

  tbody._()._();
}
Example 10
Source File: TestRMRestart.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 60000)
public void testAppAttemptTokensRestoredOnRMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  UserGroupInformation.setConfiguration(conf);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();
  MockRM rm1 = new TestSecurityMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("0.0.0.0:4321", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // submit an app
  RMApp app1 =
      rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), "default");

  // assert app info is saved
  ApplicationStateData appState = rmAppState.get(app1.getApplicationId());
  Assert.assertNotNull(appState);

  // Allocate the AM
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);

  // assert attempt info is saved
  ApplicationAttemptStateData attemptState = appState.getAttempt(attemptId1);
  Assert.assertNotNull(attemptState);
  Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
    attemptState.getMasterContainer().getId());

  // the clientTokenMasterKey that are generated when
  // RMAppAttempt is created,
  byte[] clientTokenMasterKey =
      attempt1.getClientTokenMasterKey().getEncoded();

  // assert application credentials are saved
  Credentials savedCredentials = attemptState.getAppAttemptTokens();
  Assert.assertArrayEquals("client token master key not saved",
      clientTokenMasterKey, savedCredentials.getSecretKey(
          RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));

  // start new RM
  MockRM rm2 = new TestSecurityMockRM(conf, memStore);
  rm2.start();

  RMApp loadedApp1 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId());
  RMAppAttempt loadedAttempt1 = loadedApp1.getRMAppAttempt(attemptId1);

  // assert loaded attempt recovered
  Assert.assertNotNull(loadedAttempt1);

  // assert client token master key is recovered back to api-versioned
  // client token master key
  Assert.assertEquals("client token master key not restored",
      attempt1.getClientTokenMasterKey(),
      loadedAttempt1.getClientTokenMasterKey());

  // assert ClientTokenSecretManager also knows about the key
  Assert.assertArrayEquals(clientTokenMasterKey,
      rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
          .getEncoded());

  // assert AMRMTokenSecretManager also knows about the AMRMToken password
  Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
  Assert.assertArrayEquals(amrmToken.getPassword(),
    rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
      amrmToken.decodeIdentifier()));
}
Example 11
Source File: RMAppBlock.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
protected void createApplicationAttemptTable(Block html,
    Collection<ApplicationAttemptReport> attempts) {
  TBODY<TABLE<Hamlet>> tbody =
      html.table("#attempts").thead().tr().th(".id", "Attempt ID")
        .th(".started", "Started").th(".node", "Node").th(".logs", "Logs")
        ._()._().tbody();
  RMApp rmApp = this.rm.getRMContext().getRMApps().get(this.appID);
  if (rmApp == null) {
    return;
  }
  StringBuilder attemptsTableData = new StringBuilder("[\n");
  for (final ApplicationAttemptReport appAttemptReport : attempts) {
    RMAppAttempt rmAppAttempt =
        rmApp.getRMAppAttempt(appAttemptReport.getApplicationAttemptId());
    if (rmAppAttempt == null) {
      continue;
    }
    AppAttemptInfo attemptInfo =
        new AppAttemptInfo(rmAppAttempt, rmApp.getUser());
    String nodeLink = attemptInfo.getNodeHttpAddress();
    if (nodeLink != null) {
      nodeLink = WebAppUtils.getHttpSchemePrefix(conf) + nodeLink;
    }
    String logsLink = attemptInfo.getLogsLink();
    attemptsTableData
      .append("[\"<a href='")
      .append(url("appattempt", rmAppAttempt.getAppAttemptId().toString()))
      .append("'>")
      .append(String.valueOf(rmAppAttempt.getAppAttemptId()))
      .append("</a>\",\"")
      .append(attemptInfo.getStartTime())
      .append("\",\"<a ")
      .append(nodeLink == null ? "#" : "href='" + nodeLink)
      .append("'>")
      .append(
        nodeLink == null ? "N/A" : StringEscapeUtils
          .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))
      .append("</a>\",\"<a ")
      .append(logsLink == null ? "#" : "href='" + logsLink).append("'>")
      .append(logsLink == null ? "N/A" : "Logs").append("</a>\"],\n");
  }
  if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
    attemptsTableData.delete(attemptsTableData.length() - 2,
      attemptsTableData.length() - 1);
  }
  attemptsTableData.append("]");
  html.script().$type("text/javascript")
    ._("var attemptsTableData=" + attemptsTableData)._();

  tbody._()._();
}
Example 12
Source File: TestRMRestart.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 60000)
public void testAppAttemptTokensRestoredOnRMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  UserGroupInformation.setConfiguration(conf);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();
  MockRM rm1 = new TestSecurityMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("0.0.0.0:4321", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // submit an app
  RMApp app1 =
      rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), "default");

  // assert app info is saved
  ApplicationStateData appState = rmAppState.get(app1.getApplicationId());
  Assert.assertNotNull(appState);

  // Allocate the AM
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);

  // assert attempt info is saved
  ApplicationAttemptStateData attemptState = appState.getAttempt(attemptId1);
  Assert.assertNotNull(attemptState);
  Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
    attemptState.getMasterContainer().getId());

  // the clientTokenMasterKey that are generated when
  // RMAppAttempt is created,
  byte[] clientTokenMasterKey =
      attempt1.getClientTokenMasterKey().getEncoded();

  // assert application credentials are saved
  Credentials savedCredentials = attemptState.getAppAttemptTokens();
  Assert.assertArrayEquals("client token master key not saved",
      clientTokenMasterKey, savedCredentials.getSecretKey(
          RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));

  // start new RM
  MockRM rm2 = new TestSecurityMockRM(conf, memStore);
  rm2.start();

  RMApp loadedApp1 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId());
  RMAppAttempt loadedAttempt1 = loadedApp1.getRMAppAttempt(attemptId1);

  // assert loaded attempt recovered
  Assert.assertNotNull(loadedAttempt1);

  // assert client token master key is recovered back to api-versioned
  // client token master key
  Assert.assertEquals("client token master key not restored",
      attempt1.getClientTokenMasterKey(),
      loadedAttempt1.getClientTokenMasterKey());

  // assert ClientTokenSecretManager also knows about the key
  Assert.assertArrayEquals(clientTokenMasterKey,
      rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
          .getEncoded());

  // assert AMRMTokenSecretManager also knows about the AMRMToken password
  Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
  Assert.assertArrayEquals(amrmToken.getPassword(),
    rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
      amrmToken.decodeIdentifier()));
}