org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SystemMetricsPublisher.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public void appAttemptFinished(RMAppAttempt appAttempt,
    RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
  if (publishSystemMetrics) {
    dispatcher.getEventHandler().handle(
        new AppAttemptFinishedEvent(
            appAttempt.getAppAttemptId(),
            appAttempt.getTrackingUrl(),
            appAttempt.getOriginalTrackingUrl(),
            appAttempt.getDiagnostics(),
            // app will get the final status from app attempt, or create one
            // based on app state if it doesn't exist
            app.getFinalApplicationStatus(),
            RMServerUtils.createApplicationAttemptState(appAttemtpState),
            finishedTime));
  }
}
 
Example #2
Source File: TestRMWebServicesApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id,
    long startTime, String containerId, String nodeHttpAddress, String nodeId,
    String logsLink, String user)
        throws JSONException, Exception {

  assertEquals("id doesn't match", appAttempt.getAppAttemptId()
      .getAttemptId(), id);
  assertEquals("startedTime doesn't match", appAttempt.getStartTime(),
      startTime);
  WebServicesTestUtils.checkStringMatch("containerId", appAttempt
      .getMasterContainer().getId().toString(), containerId);
  WebServicesTestUtils.checkStringMatch("nodeHttpAddress", appAttempt
      .getMasterContainer().getNodeHttpAddress(), nodeHttpAddress);
  WebServicesTestUtils.checkStringMatch("nodeId", appAttempt
      .getMasterContainer().getNodeId().toString(), nodeId);
  assertTrue("logsLink doesn't match", logsLink.startsWith("//"));
  assertTrue(
      "logsLink doesn't contain user info", logsLink.endsWith("/"
      + user));
}
 
Example #3
Source File: RMStateStoreTestBase.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected ContainerId storeAttempt(RMStateStore store,
    ApplicationAttemptId attemptId,
    String containerIdStr, Token<AMRMTokenIdentifier> appToken,
    SecretKey clientTokenMasterKey, TestDispatcher dispatcher)
    throws Exception {

  RMAppAttemptMetrics mockRmAppAttemptMetrics = 
      mock(RMAppAttemptMetrics.class);
  Container container = new ContainerPBImpl();
  container.setId(ConverterUtils.toContainerId(containerIdStr));
  RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
  when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
  when(mockAttempt.getMasterContainer()).thenReturn(container);
  when(mockAttempt.getAMRMToken()).thenReturn(appToken);
  when(mockAttempt.getClientTokenMasterKey())
      .thenReturn(clientTokenMasterKey);
  when(mockAttempt.getRMAppAttemptMetrics())
      .thenReturn(mockRmAppAttemptMetrics);
  when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
      .thenReturn(new AggregateAppResourceUsage(0, 0, 0));
  dispatcher.attemptId = attemptId;
  store.storeNewApplicationAttempt(mockAttempt);
  waitNotify(dispatcher);
  return container.getId();
}
 
Example #4
Source File: AppAttemptInfo.java    From big-c with Apache License 2.0 6 votes vote down vote up
public AppAttemptInfo(RMAppAttempt attempt, String user) {
  this.startTime = 0;
  this.containerId = "";
  this.nodeHttpAddress = "";
  this.nodeId = "";
  this.logsLink = "";
  if (attempt != null) {
    this.id = attempt.getAppAttemptId().getAttemptId();
    this.startTime = attempt.getStartTime();
    Container masterContainer = attempt.getMasterContainer();
    if (masterContainer != null) {
      this.containerId = masterContainer.getId().toString();
      this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
      this.nodeId = masterContainer.getNodeId().toString();
      this.logsLink =
          WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()), user);
    }
  }
}
 
Example #5
Source File: ResourceManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void handle(RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptID = event.getApplicationAttemptId();
  ApplicationId appAttemptId = appAttemptID.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appAttemptId);
  if (rmApp != null) {
    RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptID);
    if (rmAppAttempt != null) {
      try {
        rmAppAttempt.handle(event);
      } catch (Throwable t) {
        LOG.error("Error in handling event type " + event.getType()
            + " for applicationAttempt " + appAttemptId, t);
      }
    }
  }
}
 
Example #6
Source File: FairSchedulerTestBase.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected ApplicationAttemptId createSchedulingRequest(String queueId,
    String userId, List<ResourceRequest> ask) {
  ApplicationAttemptId id = createAppAttemptId(this.APP_ID++,
      this.ATTEMPT_ID++);
  scheduler.addApplication(id.getApplicationId(), queueId, userId, false);
  // This conditional is for testAclSubmitApplication where app is rejected
  // and no app is added.
  if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) {
    scheduler.addApplicationAttempt(id, false, false);
  }

  RMApp rmApp = mock(RMApp.class);
  RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
  when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
  when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn(
      new RMAppAttemptMetrics(id,resourceManager.getRMContext()));
  resourceManager.getRMContext().getRMApps()
      .put(id.getApplicationId(), rmApp);

  scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null);
  return id;
}
 
Example #7
Source File: ResourceManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void handle(RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptID = event.getApplicationAttemptId();
  ApplicationId appAttemptId = appAttemptID.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appAttemptId);
  if (rmApp != null) {
    RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptID);
    if (rmAppAttempt != null) {
      try {
        rmAppAttempt.handle(event);
      } catch (Throwable t) {
        LOG.error("Error in handling event type " + event.getType()
            + " for applicationAttempt " + appAttemptId, t);
      }
    }
  }
}
 
Example #8
Source File: TestRMWebServicesApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt,
    String user)
    throws JSONException, Exception {

  for (int i = 0; i < nodes.getLength(); i++) {
    Element element = (Element) nodes.item(i);

    verifyAppAttemptInfoGeneric(appAttempt,
        WebServicesTestUtils.getXmlInt(element, "id"),
        WebServicesTestUtils.getXmlLong(element, "startTime"),
        WebServicesTestUtils.getXmlString(element, "containerId"),
        WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
        WebServicesTestUtils.getXmlString(element, "nodeId"),
        WebServicesTestUtils.getXmlString(element, "logsLink"), user);
  }
}
 
Example #9
Source File: RMContainerImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static void updateAttemptMetrics(RMContainerImpl container) {
  // If this is a preempted container, update preemption metrics
  Resource resource = container.getContainer().getResource();
  RMAppAttempt rmAttempt = container.rmContext.getRMApps()
      .get(container.getApplicationAttemptId().getApplicationId())
      .getCurrentAppAttempt();
  if (ContainerExitStatus.PREEMPTED == container.finishedStatus
    .getExitStatus()) {
    rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
      container);
  }

  if (rmAttempt != null) {
    long usedMillis = container.finishTime - container.creationTime;
    long memorySeconds = resource.getMemory()
                          * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long vcoreSeconds = resource.getVirtualCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long gcoreSeconds = resource.getGpuCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    rmAttempt.getRMAppAttemptMetrics()
              .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds, gcoreSeconds);
  }
}
 
Example #10
Source File: RMAppImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public float getProgress() {
  RMAppAttempt attempt = this.currentAttempt;
  if (attempt != null) {
    return attempt.getProgress();
  }
  return 0;
}
 
Example #11
Source File: RMAppImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public RMAppMetrics getRMAppMetrics() {
  Resource resourcePreempted = Resource.newInstance(0, 0);
  int numAMContainerPreempted = 0;
  int numNonAMContainerPreempted = 0;
  long memorySeconds = 0;
  long vcoreSeconds = 0;
  for (RMAppAttempt attempt : attempts.values()) {
    if (null != attempt) {
      RMAppAttemptMetrics attemptMetrics =
          attempt.getRMAppAttemptMetrics();
      Resources.addTo(resourcePreempted,
          attemptMetrics.getResourcePreempted());
      numAMContainerPreempted += attemptMetrics.getIsPreempted() ? 1 : 0;
      numNonAMContainerPreempted +=
          attemptMetrics.getNumNonAMContainersPreempted();
      // getAggregateAppResourceUsage() will calculate resource usage stats
      // for both running and finished containers.
      AggregateAppResourceUsage resUsage =
          attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
      memorySeconds += resUsage.getMemorySeconds();
      vcoreSeconds += resUsage.getVcoreSeconds();
    }
  }

  return new RMAppMetrics(resourcePreempted,
      numNonAMContainerPreempted, numAMContainerPreempted,
      memorySeconds, vcoreSeconds);
}
 
Example #12
Source File: ClientRMService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationAttemptsResponse getApplicationAttempts(
    GetApplicationAttemptsRequest request) throws YarnException, IOException {
  ApplicationId appId = request.getApplicationId();
  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    throw RPCUtil.getRemoteException(ie);
  }
  RMApp application = this.rmContext.getRMApps().get(appId);
  if (application == null) {
    // If the RM doesn't have the application, throw
    // ApplicationNotFoundException and let client to handle.
    throw new ApplicationNotFoundException("Application with id '" + appId
        + "' doesn't exist in RM.");
  }
  boolean allowAccess = checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.VIEW_APP, application);
  GetApplicationAttemptsResponse response = null;
  if (allowAccess) {
    Map<ApplicationAttemptId, RMAppAttempt> attempts = application
        .getAppAttempts();
    List<ApplicationAttemptReport> listAttempts = 
      new ArrayList<ApplicationAttemptReport>();
    Iterator<Map.Entry<ApplicationAttemptId, RMAppAttempt>> iter = attempts
        .entrySet().iterator();
    while (iter.hasNext()) {
      listAttempts.add(iter.next().getValue()
          .createApplicationAttemptReport());
    }
    response = GetApplicationAttemptsResponse.newInstance(listAttempts);
  } else {
    throw new YarnException("User " + callerUGI.getShortUserName()
        + " does not have privilage to see this aplication " + appId);
  }
  return response;
}
 
Example #13
Source File: RMHATestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
    throws Exception {
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  nm.nodeHeartbeat(true);
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
  rm.waitForState(app.getCurrentAppAttempt().getAppAttemptId(),
      RMAppAttemptState.RUNNING);
  return am;
}
 
Example #14
Source File: ResourceTrackerService.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
 
Example #15
Source File: RMAppImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void createNewAttempt() {
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, attempts.size() + 1);
  RMAppAttempt attempt =
      new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
        submissionContext, conf,
        // The newly created attempt maybe last attempt if (number of
        // previously failed attempts(which should not include Preempted,
        // hardware error and NM resync) + 1) equal to the max-attempt
        // limit.
        maxAppAttempts == (getNumFailedAppAttempts() + 1), amReq);
  attempts.put(appAttemptId, attempt);
  currentAttempt = attempt;
}
 
Example #16
Source File: RMAppImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void createNewAttempt() {
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, attempts.size() + 1);
  RMAppAttempt attempt =
      new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
        submissionContext, conf,
        // The newly created attempt maybe last attempt if (number of
        // previously failed attempts(which should not include Preempted,
        // hardware error and NM resync) + 1) equal to the max-attempt
        // limit.
        maxAppAttempts == (getNumFailedAppAttempts() + 1), amReq);
  attempts.put(appAttemptId, attempt);
  currentAttempt = attempt;
}
 
Example #17
Source File: RMAppImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public float getProgress() {
  RMAppAttempt attempt = this.currentAttempt;
  if (attempt != null) {
    return attempt.getProgress();
  }
  return 0;
}
 
Example #18
Source File: TestAMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void waitForContainersToFinish(int expectedNum, RMAppAttempt attempt)
    throws InterruptedException {
  int count = 0;
  while (attempt.getJustFinishedContainers().size() != expectedNum
      && count < 500) {
    Thread.sleep(100);
    count++;
  }
}
 
Example #19
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
public void testAMContainerStatusWithRMRestart() throws Exception {  
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1_1 = rm1.submitApp(1024);
  MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
  
  RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt();
  AbstractYarnScheduler scheduler =
      ((AbstractYarnScheduler) rm1.getResourceScheduler());
  
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());

  // Re-start RM
  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());

  List<NMContainerStatus> am1_1Containers =
      createNMContainerStatusForApp(am1_1);
  nm1.registerNode(am1_1Containers, null);

  // Wait for RM to settle down on recovering containers;
  waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId());

  scheduler = ((AbstractYarnScheduler) rm2.getResourceScheduler());
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());
}
 
Example #20
Source File: SystemMetricsPublisher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public void appAttemptRegistered(RMAppAttempt appAttempt,
    long registeredTime) {
  if (publishSystemMetrics) {
    dispatcher.getEventHandler().handle(
        new AppAttemptRegisteredEvent(
            appAttempt.getAppAttemptId(),
            appAttempt.getHost(),
            appAttempt.getRpcPort(),
            appAttempt.getTrackingUrl(),
            appAttempt.getOriginalTrackingUrl(),
            appAttempt.getMasterContainer().getId(),
            registeredTime));
  }
}
 
Example #21
Source File: RMAppManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * create a summary of the application's runtime.
 * 
 * @param app {@link RMApp} whose summary is to be created, cannot
 *            be <code>null</code>.
 */
public static SummaryBuilder createAppSummary(RMApp app) {
  String trackingUrl = "N/A";
  String host = "N/A";
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  if (attempt != null) {
    trackingUrl = attempt.getTrackingUrl();
    host = attempt.getHost();
  }
  RMAppMetrics metrics = app.getRMAppMetrics();
  SummaryBuilder summary = new SummaryBuilder()
      .add("appId", app.getApplicationId())
      .add("name", app.getName())
      .add("user", app.getUser())
      .add("queue", app.getQueue())
      .add("state", app.getState())
      .add("trackingUrl", trackingUrl)
      .add("appMasterHost", host)
      .add("startTime", app.getStartTime())
      .add("finishTime", app.getFinishTime())
      .add("finalStatus", app.getFinalApplicationStatus())
      .add("memorySeconds", metrics.getMemorySeconds())
      .add("vcoreSeconds", metrics.getVcoreSeconds())
      .add("gcoreSeconds", metrics.getGcoreSeconds())
      .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
      .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
      .add("preemptedResources", metrics.getResourcePreempted())
      .add("applicationType", app.getApplicationType());
  return summary;
}
 
Example #22
Source File: TestRMWebServicesApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt,
    String user)
    throws JSONException, Exception {

  assertEquals("incorrect number of elements", 6, info.length());

  verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
      info.getLong("startTime"), info.getString("containerId"),
      info.getString("nodeHttpAddress"), info.getString("nodeId"),
      info.getString("logsLink"), user);
}
 
Example #23
Source File: RMContainerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static void updateAttemptMetrics(RMContainerImpl container) {
  // If this is a preempted container, update preemption metrics
  Resource resource = container.getContainer().getResource();
  RMAppAttempt rmAttempt = container.rmContext.getRMApps()
      .get(container.getApplicationAttemptId().getApplicationId())
      .getCurrentAppAttempt();
  if (ContainerExitStatus.PREEMPTED == container.finishedStatus
    .getExitStatus()) {
    rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
      container);
  }
  
  if (rmAttempt != null) {
    long usedMillis = container.finishTime - container.creationTime;
    long memorySeconds = (long)(resource.getMemory()*container.utilization)
                          * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long vcoreSeconds = (long)(resource.getVirtualCores()*container.utilization)
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    
    if (container.suspendTime.size() >0 && container.resumeTime.size() >0 && container.suspendTime.size() == container.resumeTime.size()){
    	double acc=0;
    	for(int i=0; i < container.suspendTime.size();i++){
    		
    		acc = acc + (container.resumeTime.get(i) - container.suspendTime.get(i));
    	}
    	container.utilization = acc/usedMillis;  	
    }
    rmAttempt.getRMAppAttemptMetrics()
              .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds);
  }
}
 
Example #24
Source File: ApplicationMasterLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void  handle(AMLauncherEvent appEvent) {
  AMLauncherEventType event = appEvent.getType();
  RMAppAttempt application = appEvent.getAppAttempt();
  switch (event) {
  case LAUNCH:
    launch(application);
    break;
  case CLEANUP:
    cleanup(application);
    break;
  default:
    break;
  }
}
 
Example #25
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAllocateContainerOnNodeWithoutOffSwitchSpecified()
    throws Exception {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  
  MockRM rm = new MockRM(conf);
  rm.start();
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);

  RMApp app1 = rm.submitApp(2048);
  // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();

  // add request for containers
  List<ResourceRequest> requests = new ArrayList<ResourceRequest>();
  requests.add(am1.createResourceReq("127.0.0.1", 1 * GB, 1, 1));
  requests.add(am1.createResourceReq("/default-rack", 1 * GB, 1, 1));
  am1.allocate(requests, null); // send the request

  try {
    // kick the schedule
    nm1.nodeHeartbeat(true);
  } catch (NullPointerException e) {
    Assert.fail("NPE when allocating container on node but "
        + "forget to set off-switch request should be handled");
  }
  rm.stop();
}
 
Example #26
Source File: RMStateStore.java    From big-c with Apache License 2.0 5 votes vote down vote up
public Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
  Credentials credentials = new Credentials();

  SecretKey clientTokenMasterKey =
      appAttempt.getClientTokenMasterKey();
  if(clientTokenMasterKey != null){
    credentials.addSecretKey(AM_CLIENT_TOKEN_MASTER_KEY_NAME,
        clientTokenMasterKey.getEncoded());
  }
  return credentials;
}
 
Example #27
Source File: TestRMAppTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testUnmanagedApp() throws IOException {
  ApplicationSubmissionContext subContext = new ApplicationSubmissionContextPBImpl();
  subContext.setUnmanagedAM(true);

  // test success path
  LOG.info("--- START: testUnmanagedAppSuccessPath ---");
  final String diagMsg = "some diagnostics";
  RMApp application = testCreateAppFinished(subContext, diagMsg);
  Assert.assertTrue("Finished app missing diagnostics",
      application.getDiagnostics().indexOf(diagMsg) != -1);

  // reset the counter of Mockito.verify
  reset(writer);
  reset(publisher);

  // test app fails after 1 app attempt failure
  LOG.info("--- START: testUnmanagedAppFailPath ---");
  application = testCreateAppRunning(subContext);
  RMAppEvent event = new RMAppFailedAttemptEvent(
      application.getApplicationId(), RMAppEventType.ATTEMPT_FAILED, "", false);
  application.handle(event);
  rmDispatcher.await();
  RMAppAttempt appAttempt = application.getCurrentAppAttempt();
  Assert.assertEquals(1, appAttempt.getAppAttemptId().getAttemptId());
  sendAppUpdateSavedEvent(application);
  assertFailed(application,
      ".*Unmanaged application.*Failing the application.*");
  assertAppFinalStateSaved(application);
}
 
Example #28
Source File: TestAMAuthorization.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void waitForLaunchedState(RMAppAttempt attempt)
    throws InterruptedException {
  int waitCount = 0;
  while (attempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED
      && waitCount++ < 40) {
    LOG.info("Waiting for AppAttempt to reach LAUNCHED state. "
        + "Current state is " + attempt.getAppAttemptState());
    Thread.sleep(1000);
  }
  Assert.assertEquals(attempt.getAppAttemptState(),
      RMAppAttemptState.LAUNCHED);
}
 
Example #29
Source File: RMWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up
@GET
@Path("/apps/{appid}/appattempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) {

  init();
  if (appId == null || appId.isEmpty()) {
    throw new NotFoundException("appId, " + appId + ", is empty or null");
  }
  ApplicationId id;
  id = ConverterUtils.toApplicationId(recordFactory, appId);
  if (id == null) {
    throw new NotFoundException("appId is null");
  }
  RMApp app = rm.getRMContext().getRMApps().get(id);
  if (app == null) {
    throw new NotFoundException("app with id: " + appId + " not found");
  }

  AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo();
  for (RMAppAttempt attempt : app.getAppAttempts().values()) {
    AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
    appAttemptsInfo.add(attemptInfo);
  }

  return appAttemptsInfo;
}
 
Example #30
Source File: MockRMWithCustomAMLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected ApplicationMasterLauncher createAMLauncher() {
  return new ApplicationMasterLauncher(getRMContext()) {
    @Override
    protected Runnable createRunnableLauncher(RMAppAttempt application,
        AMLauncherEventType event) {
      return new AMLauncher(context, application, event, getConfig()) {
        @Override
        protected ContainerManagementProtocol getContainerMgrProxy(
            ContainerId containerId) {
          return containerManager;
        }
        @Override
        protected Token<AMRMTokenIdentifier> createAndSetAMRMToken() {
          Token<AMRMTokenIdentifier> amRmToken =
              super.createAndSetAMRMToken();
          InetSocketAddress serviceAddr =
              getConfig().getSocketAddr(
                YarnConfiguration.RM_SCHEDULER_ADDRESS,
                YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
                YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
          SecurityUtil.setTokenService(amRmToken, serviceAddr);
          return amRmToken;
        }
      };
    }
  };
}