Java Code Examples for org.apache.hadoop.yarn.api.records.YarnApplicationState#RUNNING

public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 

public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 

protected YarnApplicationState convertDAGAppMasterState(DAGAppMasterState dagAppMasterState) {
  switch (dagAppMasterState) {
  case NEW:
    return YarnApplicationState.NEW;
  case INITED:
  case RECOVERING:
  case IDLE:
  case RUNNING:
    return YarnApplicationState.RUNNING;
  case SUCCEEDED:
    return YarnApplicationState.FINISHED;
  case FAILED:
    return YarnApplicationState.FAILED;
  case KILLED:
    return YarnApplicationState.KILLED;
  case ERROR:
    return YarnApplicationState.FAILED;
  default:
    return YarnApplicationState.SUBMITTED;
  }
}
 

@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 

@Override
protected void doStartUp() {
  super.doStartUp();

  // Submit and poll the status of the yarn application
  try {
    processController = startUp.call();

    YarnApplicationReport report = processController.getReport();
    ApplicationId appId = report.getApplicationId();
    LOG.info("Application {} with id {} submitted", appName, appId);

    YarnApplicationState state = report.getYarnApplicationState();
    Stopwatch stopWatch = new Stopwatch().start();

    LOG.debug("Checking yarn application status for {} {}", appName, appId);
    while (!hasRun(state) && stopWatch.elapsedTime(startTimeoutUnit) < startTimeout) {
      report = processController.getReport();
      state = report.getYarnApplicationState();
      LOG.debug("Yarn application status for {} {}: {}", appName, appId, state);
      TimeUnit.SECONDS.sleep(1);
    }
    LOG.info("Yarn application {} {} is in state {}", appName, appId, state);
    if (state != YarnApplicationState.RUNNING) {
      LOG.info("Yarn application {} {} is not in running state. Shutting down controller.", appName, appId);
      forceShutDown();
    }

    currentAttemptId = report.getCurrentApplicationAttemptId();
  } catch (Exception e) {
    throw Throwables.propagate(e);
  }
}
 

@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 

/**
 * For some yet unknown reason, hostname resolution for the ResourceManager in {@link MiniYARNCluster}
 * has some issue that causes the {@link YarnClient} not be able to connect and submit the Gobblin Yarn
 * application successfully. This works fine on local machine though. So disabling this and the test
 * below that depends on it on Travis-CI.
 */
@Test(enabled=false, groups = { "disabledOnTravis" }, dependsOnMethods = "testCreateHelixCluster")
public void testSetupAndSubmitApplication() throws Exception {
  HelixUtils.createGobblinHelixCluster(
      this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY),
      this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY));

  this.gobblinYarnAppLauncher.startYarnClient();
  this.applicationId = this.gobblinYarnAppLauncher.setupAndSubmitApplication();

  int i;

  // wait for application to come up
  for (i = 0; i < 120; i++) {
    if (yarnClient.getApplicationReport(applicationId).getYarnApplicationState() ==
        YarnApplicationState.RUNNING) {
      break;
    }
    Thread.sleep(1000);
  }

  Assert.assertTrue(i < 120, "timed out waiting for RUNNING state");

  // wait another 10 seconds and check state again to make sure that the application stays up
  Thread.sleep(10000);

  Assert.assertEquals(yarnClient.getApplicationReport(applicationId).getYarnApplicationState(),
      YarnApplicationState.RUNNING, "Application may have aborted");
}
 

static DAGClientAMProtocolBlockingPB getSessionAMProxy(YarnClient yarnClient,
    Configuration conf,
    ApplicationId applicationId) throws TezException, IOException {
  ApplicationReport appReport;
  try {
    appReport = yarnClient.getApplicationReport(
        applicationId);

    if(appReport == null) {
      throw new TezUncheckedException("Could not retrieve application report"
          + " from YARN, applicationId=" + applicationId);
    }
    YarnApplicationState appState = appReport.getYarnApplicationState();
    if(appState != YarnApplicationState.RUNNING) {
      if (appState == YarnApplicationState.FINISHED
          || appState == YarnApplicationState.KILLED
          || appState == YarnApplicationState.FAILED) {
        throw new SessionNotRunning("Application not running"
            + ", applicationId=" + applicationId
            + ", yarnApplicationState=" + appReport.getYarnApplicationState()
            + ", finalApplicationStatus="
            + appReport.getFinalApplicationStatus()
            + ", trackingUrl=" + appReport.getTrackingUrl());
      }
      return null;
    }
  } catch (YarnException e) {
    throw new TezException(e);
  }
  return getAMProxy(conf, appReport.getHost(),
      appReport.getRpcPort(), appReport.getClientToAMToken());
}
 

boolean createAMProxyIfNeeded() throws IOException, TezException {
  if(proxy != null) {
    // if proxy exist optimistically use it assuming there is no retry
    return true;
  }
  appReport = getAppReport();

  if(appReport == null) {
    return false;
  }
  YarnApplicationState appState = appReport.getYarnApplicationState();
  if(appState != YarnApplicationState.RUNNING) {
    return false;
  }

  // YARN-808. Cannot ascertain if AM is ready until we connect to it.
  // workaround check the default string set by YARN
  if(appReport.getHost() == null || appReport.getHost().equals("N/A") ||
      appReport.getRpcPort() == 0){
    // attempt not running
    return false;
  }

  proxy = TezClientUtils.getAMProxy(conf, appReport.getHost(), appReport.getRpcPort(),
      appReport.getClientToAMToken());
  return true;
}
 

boolean createAMProxyIfNeeded() throws IOException, TezException,
    ApplicationNotFoundException {
  if(proxy != null) {
    // if proxy exist optimistically use it assuming there is no retry
    return true;
  }
  appReport = null;
  appReport = getAppReport();

  if(appReport == null) {
    return false;
  }
  YarnApplicationState appState = appReport.getYarnApplicationState();
  if(appState != YarnApplicationState.RUNNING) {
    return false;
  }

  // YARN-808. Cannot ascertain if AM is ready until we connect to it.
  // workaround check the default string set by YARN
  if(appReport.getHost() == null || appReport.getHost().equals("N/A") ||
      appReport.getRpcPort() == 0){
    // attempt not running
    return false;
  }

  proxy = TezClientUtils.getAMProxy(conf, appReport.getHost(), appReport.getRpcPort(),
      appReport.getClientToAMToken(), ugi);
  return true;
}
 

@Override
public void execute(String[] args, ConsoleReader reader) throws Exception
{
  try {
    JSONArray jsonArray = new JSONArray();
    List<ApplicationReport> appList = getApplicationList();
    Collections.sort(appList, new Comparator<ApplicationReport>()
    {
      @Override
      public int compare(ApplicationReport o1, ApplicationReport o2)
      {
        return o1.getApplicationId().getId() - o2.getApplicationId().getId();
      }

    });
    int totalCnt = 0;
    int runningCnt = 0;

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z");

    for (ApplicationReport ar : appList) {
      /*
       * This is inefficient, but what the heck, if this can be passed through the command line, can anyone notice slowness.
       */
      JSONObject jsonObj = new JSONObject();
      jsonObj.put("startTime", sdf.format(new java.util.Date(ar.getStartTime())));
      jsonObj.put("id", ar.getApplicationId().getId());
      jsonObj.put("name", ar.getName());
      jsonObj.put("state", ar.getYarnApplicationState().name());
      jsonObj.put("trackingUrl", ar.getTrackingUrl());
      jsonObj.put("finalStatus", ar.getFinalApplicationStatus());
      JSONArray tags = new JSONArray();
      for (String tag : ar.getApplicationTags()) {
        tags.put(tag);
      }
      jsonObj.put("tags", tags);

      totalCnt++;
      if (ar.getYarnApplicationState() == YarnApplicationState.RUNNING) {
        runningCnt++;
      }

      if (args.length > 1) {
        if (StringUtils.isNumeric(args[1])) {
          if (jsonObj.getString("id").equals(args[1])) {
            jsonArray.put(jsonObj);
            break;
          }
        } else {
          @SuppressWarnings("unchecked")
          Iterator<String> keys = jsonObj.keys();
          while (keys.hasNext()) {
            if (jsonObj.get(keys.next()).toString().toLowerCase().contains(args[1].toLowerCase())) {
              jsonArray.put(jsonObj);
              break;
            }
          }
        }
      } else {
        jsonArray.put(jsonObj);
      }
    }
    printJson(jsonArray, "apps");
    if (consolePresent) {
      System.out.println(runningCnt + " active, total " + totalCnt + " applications.");
    }
  } catch (Exception ex) {
    throw new CliException("Failed to retrieve application list", ex);
  }
}
 

@Override
public void execute(String[] args, ConsoleReader reader) throws Exception
{
  try {
    JSONArray jsonArray = new JSONArray();
    List<ApplicationReport> appList = getApplicationList();
    Collections.sort(appList, new Comparator<ApplicationReport>()
    {
      @Override
      public int compare(ApplicationReport o1, ApplicationReport o2)
      {
        return o1.getApplicationId().getId() - o2.getApplicationId().getId();
      }

    });
    int totalCnt = 0;
    int runningCnt = 0;

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z");

    for (ApplicationReport ar : appList) {
      /*
       * This is inefficient, but what the heck, if this can be passed through the command line, can anyone notice slowness.
       */
      JSONObject jsonObj = new JSONObject();
      jsonObj.put("startTime", sdf.format(new java.util.Date(ar.getStartTime())));
      jsonObj.put("id", ar.getApplicationId().getId());
      jsonObj.put("name", ar.getName());
      jsonObj.put("state", ar.getYarnApplicationState().name());
      jsonObj.put("trackingUrl", ar.getTrackingUrl());
      jsonObj.put("finalStatus", ar.getFinalApplicationStatus());
      JSONArray tags = new JSONArray();
      for (String tag : ar.getApplicationTags()) {
        tags.put(tag);
      }
      jsonObj.put("tags", tags);

      totalCnt++;
      if (ar.getYarnApplicationState() == YarnApplicationState.RUNNING) {
        runningCnt++;
      }

      if (args.length > 1) {
        if (StringUtils.isNumeric(args[1])) {
          if (jsonObj.getString("id").equals(args[1])) {
            jsonArray.put(jsonObj);
            break;
          }
        } else {
          @SuppressWarnings("unchecked")
          Iterator<String> keys = jsonObj.keys();
          while (keys.hasNext()) {
            if (jsonObj.get(keys.next()).toString().toLowerCase().contains(args[1].toLowerCase())) {
              jsonArray.put(jsonObj);
              break;
            }
          }
        }
      } else {
        jsonArray.put(jsonObj);
      }
    }
    printJson(jsonArray, "apps");
    if (consolePresent) {
      System.out.println(runningCnt + " active, total " + totalCnt + " applications.");
    }
  } catch (Exception ex) {
    throw new CliException("Failed to retrieve application list", ex);
  }
}
 

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId)
        throws YarnException, IOException {

  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    // Get application report for the appId we are interested in
    ApplicationReport report = yarnClient.getApplicationReport(appId);

    LOG.info("Got application report from ASM for"
            + ", appId=" + appId.getId()
            + ", clientToAMToken=" + report.getClientToAMToken()
            + ", appDiagnostics=" + report.getDiagnostics()
            + ", appMasterHost=" + report.getHost()
            + ", appQueue=" + report.getQueue()
            + ", appMasterRpcPort=" + report.getRpcPort()
            + ", appStartTime=" + report.getStartTime()
            + ", yarnAppState=" + report.getYarnApplicationState().toString()
            + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
            + ", appTrackingUrl=" + report.getTrackingUrl()
            + ", appUser=" + report.getUser());

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if(YarnApplicationState.RUNNING == state) {
      LOG.info("Application is running...");
      return true;
    }
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;
      }
      else {
        LOG.info("Application did finished unsuccessfully."
                + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
                + ". Breaking monitoring loop");
        return false;
      }
    }
    else if (YarnApplicationState.KILLED == state
            || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
              + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
              + ". Breaking monitoring loop");
      return false;
    }

    if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
      LOG.info("Reached client specified timeout for application. Killing application");
      forceKillApplication(appId);
      return false;
    }
  }

}