Java Code Examples for org.apache.hadoop.yarn.client.api.YarnClient#getApplications()

The following examples show how to use org.apache.hadoop.yarn.client.api.YarnClient#getApplications() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StramClientUtils.java    From Bats with Apache License 2.0 6 votes vote down vote up 
public static ApplicationReport getStartedAppInstanceByName(YarnClient clientRMService, String appName, String user, String excludeAppId) throws YarnException, IOException
{
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED), EnumSet.of(YarnApplicationState.RUNNING,
      YarnApplicationState.ACCEPTED,
      YarnApplicationState.NEW,
      YarnApplicationState.NEW_SAVING,
      YarnApplicationState.SUBMITTED));
  // see whether there is an app with the app name and user name running
  for (ApplicationReport app : applications) {
    if (!app.getApplicationId().toString().equals(excludeAppId)
        && app.getName().equals(appName)
        && app.getUser().equals(user)) {
      return app;
    }
  }
  return null;
}
Example 2
Source File: StramClientUtils.java    From Bats with Apache License 2.0 6 votes vote down vote up 
public static List<ApplicationReport> cleanAppDirectories(YarnClient clientRMService, Configuration conf, FileSystem fs, long finishedBefore)
    throws IOException, YarnException
{
  List<ApplicationReport> result = new ArrayList<>();
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED),
      EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.FINISHED, YarnApplicationState.KILLED));
  Path appsBasePath = new Path(StramClientUtils.getApexDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
  for (ApplicationReport ar : applications) {
    long finishTime = ar.getFinishTime();
    if (finishTime < finishedBefore) {
      try {
        Path appPath = new Path(appsBasePath, ar.getApplicationId().toString());
        if (fs.isDirectory(appPath)) {
          LOG.debug("Deleting finished application data for {}", ar.getApplicationId());
          fs.delete(appPath, true);
          result.add(ar);
        }
      } catch (Exception ex) {
        LOG.warn("Cannot delete application data for {}", ar.getApplicationId(), ex);
        continue;
      }
    }
  }
  return result;
}
Example 3
Source File: TestRMFailover.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    client.start();
    try {
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
Example 4
Source File: ProtocolHATestBase.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
protected void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = createAndStartYarnClient(conf);
    try {
      Thread.sleep(100);
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e.getMessage());
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
Example 5
Source File: TestRMFailover.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    client.start();
    try {
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
Example 6
Source File: ProtocolHATestBase.java    From big-c with Apache License 2.0 6 votes vote down vote up 
protected void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = createAndStartYarnClient(conf);
    try {
      Thread.sleep(100);
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e.getMessage());
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
Example 7
Source File: StramClientUtils.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up 
public static ApplicationReport getStartedAppInstanceByName(YarnClient clientRMService, String appName, String user, String excludeAppId) throws YarnException, IOException
{
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED), EnumSet.of(YarnApplicationState.RUNNING,
      YarnApplicationState.ACCEPTED,
      YarnApplicationState.NEW,
      YarnApplicationState.NEW_SAVING,
      YarnApplicationState.SUBMITTED));
  // see whether there is an app with the app name and user name running
  for (ApplicationReport app : applications) {
    if (!app.getApplicationId().toString().equals(excludeAppId)
        && app.getName().equals(appName)
        && app.getUser().equals(user)) {
      return app;
    }
  }
  return null;
}
Example 8
Source File: StramClientUtils.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up 
public static List<ApplicationReport> cleanAppDirectories(YarnClient clientRMService, Configuration conf, FileSystem fs, long finishedBefore)
    throws IOException, YarnException
{
  List<ApplicationReport> result = new ArrayList<>();
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED),
      EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.FINISHED, YarnApplicationState.KILLED));
  Path appsBasePath = new Path(StramClientUtils.getApexDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
  for (ApplicationReport ar : applications) {
    long finishTime = ar.getFinishTime();
    if (finishTime < finishedBefore) {
      try {
        Path appPath = new Path(appsBasePath, ar.getApplicationId().toString());
        if (fs.isDirectory(appPath)) {
          LOG.debug("Deleting finished application data for {}", ar.getApplicationId());
          fs.delete(appPath, true);
          result.add(ar);
        }
      } catch (Exception ex) {
        LOG.warn("Cannot delete application data for {}", ar.getApplicationId(), ex);
        continue;
      }
    }
  }
  return result;
}
Example 9
Source File: YARNSessionCapacitySchedulerITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private ApplicationReport getOnlyApplicationReport() throws IOException, YarnException {
	final YarnClient yarnClient = getYarnClient();
	checkState(yarnClient != null);

	final List<ApplicationReport> apps = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
	assertEquals(1, apps.size()); // Only one running
	return apps.get(0);
}
Example 10
Source File: YARNSessionCapacitySchedulerITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private ApplicationReport getOnlyApplicationReport() throws IOException, YarnException {
	final YarnClient yarnClient = getYarnClient();
	checkState(yarnClient != null);

	final List<ApplicationReport> apps = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
	assertEquals(1, apps.size()); // Only one running
	return apps.get(0);
}
Example 11
Source File: YarnTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected ApplicationReport getOnlyApplicationReport() throws IOException, YarnException {
	final YarnClient yarnClient = getYarnClient();
	checkState(yarnClient != null);

	final List<ApplicationReport> apps = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
	assertEquals(1, apps.size()); // Only one running
	return apps.get(0);
}
Example 12
Source File: OlapServerSubmitter.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up 
private ApplicationId findApplication(YarnClient yarnClient) throws IOException, YarnException {
    List<ApplicationReport> reports = yarnClient.getApplications(Collections.singleton("YARN"),
            EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING));
    for (ApplicationReport ar : reports) {
        if (ar.getName().equals("OlapServer-"+queueName) && ar.getUser().equals(UserGroupInformation.getCurrentUser().getUserName())) {
            LOG.info("Found existing application: " + ar);
            return ar.getApplicationId();
        }
    }
    return null;
}
Example 13
Source File: TestTezJobs.java    From tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testSessionTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/sessiontimeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testSessionTimeout", tezConf, true);
    tezClient.start();

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Session timed out"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}
Example 14
Source File: StramUtils.java    From Bats with Apache License 2.0 4 votes vote down vote up 
public static List<ApplicationReport> getApexApplicationList(YarnClient yarnClient) throws IOException, YarnException
{
  return yarnClient.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED));
}
Example 15
Source File: TestTezJobs.java    From tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testAMClientHeartbeatTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/timeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_AM_CLIENT_HEARTBEAT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testAMClientHeartbeatTimeout", tezConf, true);
    tezClient.start();
    tezClient.cancelAMKeepAlive(true);

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Client-to-AM Heartbeat timeout interval expired"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}
Example 16
Source File: TestTezJobs.java    From tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testSimpleSessionExample() throws Exception {
  Path stagingDirPath = new Path("/tmp/owc-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  int numIterations = 2;
  String[] inputPaths = new String[numIterations];
  String[] outputPaths = new String[numIterations];
  Path[] outputDirs = new Path[numIterations];
  for (int i=0; i<numIterations; ++i) {
    String inputDirStr = "/tmp/owc-input-" + i + "/";
    inputPaths[i] = inputDirStr;
    Path inputDir = new Path(inputDirStr);
    remoteFs.mkdirs(inputDir);
    generateOrderedWordCountInput(inputDir, remoteFs);
    String outputDirStr = "/tmp/owc-output-" + i + "/"; 
    outputPaths[i] = outputDirStr;
    Path outputDir = new Path(outputDirStr);
    outputDirs[i] = outputDir;
  }


  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
  YarnClient yarnClient = YarnClient.createYarnClient();

  try {
    
    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();
    
    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    SimpleSessionExample job = new SimpleSessionExample();
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
    Assert.assertTrue(
        "SimpleSessionExample failed",
        job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
            StringUtils.join(",", outputPaths), "2" }, null) == 0);

    for (int i=0; i<numIterations; ++i) {
      verifyOutput(outputDirs[i], remoteFs);
    }
    
    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;
    
    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);
  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }

}
Example 17
Source File: EmrClusterJob.java    From datacollector with Apache License 2.0 4 votes vote down vote up 
@Override
public Properties getJobStatus(Properties jobProps) throws IOException {
  EMRJobConfig emrJobConfig = new EMRJobConfig(jobProps);
  Utils.checkNotNull(emrJobConfig.getClusterId(), "EMR Cluster Id");
  String state;
  String message = null;
  DescribeStepResult res = getEmrClient(emrClusterConfig).describeStep(new DescribeStepRequest().withClusterId(
      emrJobConfig.getClusterId()).withStepId(emrJobConfig.getStepId()));
  StepStatus status = res.getStep().getStatus();
  ApplicationId appId = null;
  LOG.debug(Utils.format("Status of step: {} is {}", emrJobConfig.getStepId(), status.getState()));
  if ("PENDING".equals(status.getState())) {
    state = status.getState();
    if (status.getStateChangeReason() != null) {
      message = status.getStateChangeReason().getMessage();
    }
  } else if (!"COMPLETED".equals(status.getState()) && !"RUNNING".equals(status.getState())) {
    state = status.getState();
    if (status.getFailureDetails() != null) {
      message = status.getFailureDetails().getReason();
    }
  } else {
    YarnClient yarnClient = getYarnClient(emrJobConfig.getClusterId(), emrClusterConfig);
    ApplicationReport report = null;
    try {
      for (ApplicationReport applicationReport : yarnClient.getApplications()) {
        if (applicationReport.getName().contains(emrJobConfig.getUniquePrefix())) {
          appId = applicationReport.getApplicationId();
          break;
        }
      }
      if (appId != null) {
        report = yarnClient.getApplicationReport(appId);
      }
    } catch (YarnException ex) {
      throw new IOException("Failed to fetch yarn app report " + ex);
    }
    if (report != null) {
      YarnApplicationState yarnState = report.getYarnApplicationState();
      FinalApplicationStatus finalApplicationStatus = report.getFinalApplicationStatus();
      LOG.info(Utils.format("Application state for app id: {} is {} ", appId, yarnState));
      state = yarnState.name();
      if (YarnApplicationState.FINISHED == yarnState) {
        // override with final application state if yarnState is FINISHED
        state = finalApplicationStatus.name();
      }
      message = report.getDiagnostics();
    } else {
      state = "STARTING"; // a situation where step was in RUNNING but yarn application not yet created.
      message = "Yarn application not yet created";
    }
  }
  EmrState emrJobState = new EmrState();
  emrJobState.setState(state);
  emrJobState.setMessage(message);
  emrJobState.setAppId(appId != null ? appId.toString() : null);
  return emrJobState.toProperties();
}
Example 18
Source File: StramUtils.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up 
public static List<ApplicationReport> getApexApplicationList(YarnClient yarnClient) throws IOException, YarnException
{
  return yarnClient.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED));
}